Merge branch 'feature/bootloader_extra_dir' into 'master'

feat(bootloader): add the possibility to specify extra components directories

See merge request espressif/esp-idf!33433

components/bootloader/project_include.cmake

@@ -116,8 +116,13 @@ idf_build_get_property(sdkconfig SDKCONFIG)
 idf_build_get_property(python PYTHON)
 idf_build_get_property(extra_cmake_args EXTRA_CMAKE_ARGS)
 
-# We cannot pass lists are a parameter to the external project without modifying the ';' separator
+# BOOTLOADER_EXTRA_COMPONENT_DIRS may have been set by the `main` component, do not overwrite it
+idf_build_get_property(bootloader_extra_component_dirs BOOTLOADER_EXTRA_COMPONENT_DIRS)
+list(APPEND bootloader_extra_component_dirs "${CMAKE_CURRENT_LIST_DIR}")
+
+# We cannot pass lists as a parameter to the external project without modifying the ';' separator
 string(REPLACE ";" "|" BOOTLOADER_IGNORE_EXTRA_COMPONENT "${BOOTLOADER_IGNORE_EXTRA_COMPONENT}")
+string(REPLACE ";" "|" bootloader_extra_component_dirs "${bootloader_extra_component_dirs}")
 
 externalproject_add(bootloader
     SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/subproject"
@@ -127,7 +132,7 @@ externalproject_add(bootloader
     LIST_SEPARATOR |
     CMAKE_ARGS  -DSDKCONFIG=${sdkconfig} -DIDF_PATH=${idf_path} -DIDF_TARGET=${idf_target}
                 -DPYTHON_DEPS_CHECKED=1 -DPYTHON=${python}
-                -DEXTRA_COMPONENT_DIRS=${CMAKE_CURRENT_LIST_DIR}
+                -DEXTRA_COMPONENT_DIRS=${bootloader_extra_component_dirs}
                 -DPROJECT_SOURCE_DIR=${PROJECT_SOURCE_DIR}
                 -DIGNORE_EXTRA_COMPONENT=${BOOTLOADER_IGNORE_EXTRA_COMPONENT}
                 ${sign_key_arg} ${ver_key_arg}

components/bootloader/subproject/CMakeLists.txt

@@ -34,8 +34,8 @@ set(COMPONENTS
     esp_system
     newlib)
 
-# Make EXTRA_COMPONENT_DIRS variable to point to the bootloader_components directory
-# of the project being compiled
+# EXTRA_COMPONENT_DIRS can be populated with directories containing one or several components.
+# Make sure this variable contains `bootloader_components` directory of the project being compiled.
 set(PROJECT_EXTRA_COMPONENTS "${PROJECT_SOURCE_DIR}/bootloader_components")
 if(EXISTS ${PROJECT_EXTRA_COMPONENTS})
     list(APPEND EXTRA_COMPONENT_DIRS "${PROJECT_EXTRA_COMPONENTS}")

docs/en/api-guides/build-system.rst

@@ -245,7 +245,9 @@ These variables all have default values that can be overridden for custom behavi
 
 - ``COMPONENTS``: A list of component names to build into the project. Defaults to all components found in the ``COMPONENT_DIRS`` directories. Use this variable to "trim down" the project for faster build times. Note that any component which "requires" another component via the REQUIRES or PRIV_REQUIRES arguments on component registration will automatically have it added to this list, so the ``COMPONENTS`` list can be very short.
 
-- ``BOOTLOADER_IGNORE_EXTRA_COMPONENT``: A list of components, placed in ``bootloader_components/``, that should be ignored by the bootloader compilation. Use this variable if a bootloader component needs to be included conditionally inside the project.
+- ``BOOTLOADER_IGNORE_EXTRA_COMPONENT``: Optional list of components, placed in ``bootloader_components/``, that should be ignored by the bootloader compilation. Use this variable if a bootloader component needs to be included conditionally inside the project.
+
+- ``BOOTLOADER_EXTRA_COMPONENT_DIRS``: Optional list of additional directories to search for components to be compiled as part of the bootloader. Please note that this is a build property.
 
 Any paths in these variables can be absolute paths, or set relative to the project directory.
 
@@ -751,7 +753,7 @@ This mechanism is shown in the example :example:`build_system/wrappers`. Check :
 Override the Default Bootloader
 -------------------------------
 
-Thanks to the optional ``bootloader_components`` directory present in your ESP-IDf project, it is possible to override the default ESP-IDF bootloader. To do so, a new ``bootloader_components/main`` component should be defined, which will make the project directory tree look like the following:
+Thanks to the optional ``bootloader_components`` directory present in your ESP-IDF project, it is possible to override the default ESP-IDF bootloader. To do so, a new ``bootloader_components/main`` component should be defined, which will make the project directory tree look like the following:
 
     - myProject/
                  - CMakeLists.txt
@@ -765,7 +767,7 @@ Thanks to the optional ``bootloader_components`` directory present in your ESP-I
                  - build/
 
 
-Here the ``my_bootloader.c`` file becomes source code for the new bootloader, which means that it will need to perform all the required operations to set up and load the ``main`` application from flash.
+Here, the ``my_bootloader.c`` file becomes source code for the new bootloader, which means that it will need to perform all the required operations to set up and load the ``main`` application from flash.
 
 It is also possible to conditionally replace the bootloader depending on a certain condition, such as the target for example. This can be achieved thanks to the ``BOOTLOADER_IGNORE_EXTRA_COMPONENT`` CMake variable. This list can be used to tell the ESP-IDF bootloader project to ignore and not compile the given components present in ``bootloader_components``. For example, if one wants to use the default bootloader for ESP32 target, then ``myProject/CMakeLists.txt`` should look like the following::
 
@@ -781,6 +783,15 @@ It is important to note that this can also be used for any other bootloader comp
 
 See :example:`custom_bootloader/bootloader_override` for an example of overriding the default bootloader.
 
+Similarly to regular applications, it is possible to include external components, not placed in `bootloader_component`, as part of the bootloader build thanks to the build property ``BOOTLOADER_EXTRA_COMPONENT_DIRS``. It can either refer to a directory that contains several components, or refer to a single component. For example:
+
+    include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+
+    idf_build_set_property(BOOTLOADER_EXTRA_COMPONENT_DIRS "/path/to/extra/component/" APPEND)
+
+    project(main)
+
+See :example:`custom_bootloader/bootloader_extra_dir` for an example of adding extra components to the bootloader build.
 
 .. _config_only_component:
 

docs/zh_CN/api-guides/build-system.rst

@@ -245,7 +245,9 @@ ESP-IDF 适用于 Python 3.8 以上版本。
 
 - ``COMPONENTS``：要构建进项目中的组件名称列表，默认为 ``COMPONENT_DIRS`` 目录下检索到的所有组件。使用此变量可以“精简”项目以缩短构建时间。请注意，如果一个组件通过 ``COMPONENT_REQUIRES`` 指定了它依赖的另一个组件，则会自动将其添加到 ``COMPONENTS`` 中，所以 ``COMPONENTS`` 列表可能会非常短。
 
-- ``BOOTLOADER_IGNORE_EXTRA_COMPONENT``：引导加载程序编译时应忽略的组件列表，位于 ``bootloader_components/`` 目录中。使用这一变量可以将一个组件有条件地包含在项目中。
+- ``BOOTLOADER_IGNORE_EXTRA_COMPONENT``：可选组件列表，位于 ``bootloader_components/`` 目录中，引导加载程序编译时会忽略该列表中的组件。使用这一变量可以将一个组件有条件地包含在项目中。
+
+- ``BOOTLOADER_EXTRA_COMPONENT_DIRS``：可选的附加路径列表，引导加载程序编译时将从这些路径中搜索要编译的组件。注意，这是一个构建属性。
 
 以上变量中的路径可以是绝对路径，或者是相对于项目目录的相对路径。
 
@@ -781,6 +783,15 @@ KConfig.projbuild
 
 请参考 :example:`custom_bootloader/bootloader_override` 查看覆盖默认引导加载程序的示例。
 
+与常规应用程序类似，通过构建属性 ``BOOTLOADER_EXTRA_COMPONENT_DIRS`` 可以将不在 `bootloader_component` 中的外部组件作为引导加载程序的一部分进行构建。可以只引用一个组件，也可以引用包含多个组件的路径。例如：
+
+    include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+
+    idf_build_set_property(BOOTLOADER_EXTRA_COMPONENT_DIRS "/path/to/extra/component/" APPEND)
+
+    project(main)
+
+请参考示例 :example:`custom_bootloader/bootloader_extra_dir`，查看如何向引导加载程序构建过程添加额外的组件。
 
 .. _config_only_component:
 

examples/custom_bootloader/bootloader_extra_dir/CMakeLists.txt

@@ -0,0 +1,11 @@
+# For more information about build system see
+# https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/build-system.html
+# The following five lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.16)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+
+idf_build_set_property(BOOTLOADER_EXTRA_COMPONENT_DIRS "${CMAKE_CURRENT_LIST_DIR}/extra_bootloader_components/" APPEND)
+
+project(main)

examples/custom_bootloader/bootloader_extra_dir/README.md

@@ -0,0 +1,56 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C5 | ESP32-C6 | ESP32-C61 | ESP32-H2 | ESP32-P4 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- | --------- | -------- | -------- | -------- | -------- |
+
+# Bootloader extra component
+
+(See the README.md file in the upper level for more information about bootloader examples.)
+
+The purpose of this example is to show how to add a custom directory that contains a component to the bootloader build.
+
+Registering extra components for the bootloader can be done thanks to the IDF property `BOOTLOADER_EXTRA_COMPONENT_DIRS`. It can either refer to a directory that contains several components, either refer to a single component.
+
+## Usage of this example:
+
+Simply compile it:
+```
+idf.py build
+```
+
+Then flash it and open the monitor with the following command:
+```
+idf.py flash monitor
+```
+
+If everything went well, the bootloader should output the following message:
+```
+I (60) EXTRA: This function is called from an extra component
+```
+
+And finally the application will start and show the message:
+```
+User application is loaded and running.
+```
+
+## Organization of this example
+
+This project contains a `main` directory that represents an application. It also has a `bootloader_components` directory that contains a component that will be compiled and linked with the bootloader. This `bootloader_components` can contain several components, each of them would be in a different directory.
+
+The directory `extra_bootloader_components/extra_component/` contains a component that is meant to be included in the bootloader build. To do so, the CMake property `BOOTLOADER_EXTRA_COMPONENT_DIRS` is set from the `CMakeLists.txt` file.
+
+Below is a short explanation of files in the project folder.
+
+```
+├── CMakeLists.txt             Defines the `BOOTLOADER_EXTRA_COMPONENT_DIRS` property
+├── main
+│   ├── CMakeLists.txt
+│   └── main.c                 User application
+├── bootloader_components
+│   └── my_boot_hooks
+│       ├── CMakeLists.txt
+│       └── hooks.c            Implementation of the hooks to execute on boot
+├── extra_bootloader_components
+│   └── extra_component
+│       ├── CMakeLists.txt
+│       └── extra_component.c  Implementation of the extra component
+└── README.md                  This is the file you are currently reading
+```

examples/custom_bootloader/bootloader_extra_dir/bootloader_components/my_boot_hooks/CMakeLists.txt

@@ -0,0 +1,9 @@
+idf_component_register(SRCS "hooks.c"
+                       REQUIRES extra_component)
+
+# We need to force GCC to integrate this static library into the
+# bootloader link. Indeed, by default, as the hooks in the bootloader are weak,
+# the linker would just ignore the symbols in the extra. (i.e. not strictly
+# required)
+# To do so, we need to define the symbol (function) `bootloader_hooks_include`
+# within hooks.c source file.

examples/custom_bootloader/bootloader_extra_dir/bootloader_components/my_boot_hooks/hooks.c

@@ -0,0 +1,17 @@
+/*
+ * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+
+extern void bootloader_extra_dir_function(void);
+
+/* Function used to tell the linker to include this file
+ * with all its symbols.
+ */
+void bootloader_hooks_include(void){
+}
+
+void bootloader_after_init(void) {
+    bootloader_extra_dir_function();
+}

examples/custom_bootloader/bootloader_extra_dir/extra_bootloader_components/extra_component/CMakeLists.txt

@@ -0,0 +1 @@
+idf_component_register(SRCS "extra_component.c")

examples/custom_bootloader/bootloader_extra_dir/extra_bootloader_components/extra_component/extra_component.c

@@ -0,0 +1,11 @@
+/*
+ * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+#include "esp_log.h"
+
+void bootloader_extra_dir_function(void)
+{
+    ESP_LOGI("EXTRA", "This function is called from an extra component");
+}

examples/custom_bootloader/bootloader_extra_dir/main/CMakeLists.txt

@@ -0,0 +1,2 @@
+idf_component_register(SRCS "bootloader_hooks_example_main.c"
+                    INCLUDE_DIRS ".")

examples/custom_bootloader/bootloader_extra_dir/main/bootloader_hooks_example_main.c

@@ -0,0 +1,11 @@
+/*
+ * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+#include <stdio.h>
+
+void app_main(void)
+{
+    printf("User application is loaded and running.\n");
+}

examples/custom_bootloader/bootloader_extra_dir/pytest_bootloader_extra_dir.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.supported_targets
+@pytest.mark.generic
+def test_custom_bootloader_extra_component(dut: Dut) -> None:
+    dut.expect_exact('This function is called from an extra component')

tools/ci/check_examples_extra_component_dirs.sh

@@ -5,7 +5,8 @@ set -uo pipefail
 # Examples shouldn't use EXTRA_COMPONENT_DIRS, instead the dependencies should be specified in idf_component.yml files
 
 output=$(find ${IDF_PATH}/examples -name "CMakeLists.txt" -not -path "**/managed_components/**" -not -path "**/build/**")
-files=$(egrep "EXTRA_COMPONENT_DIRS" ${output} | cut -d ":" -f 1)
+# Make sure the regex doesn't match the text `BOOTLOADER_EXTRA_COMPONENT_DIRS`
+files=$(egrep "[^A-Za-Z0-9_]EXTRA_COMPONENT_DIRS" ${output} | cut -d ":" -f 1)
 found_issues=0
 for file in ${files}
 do