From 65d08698e82b0f00c4c8f20f3c6e537ca3d2f816 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 7 May 2024 11:02:39 +0200 Subject: [PATCH 1/3] ci: improve get_mr_components, take more components into consideration --- tools/ci/ci_get_mr_info.py | 51 +++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/tools/ci/ci_get_mr_info.py b/tools/ci/ci_get_mr_info.py index 95a15a368e..94ba708059 100644 --- a/tools/ci/ci_get_mr_info.py +++ b/tools/ci/ci_get_mr_info.py @@ -3,10 +3,9 @@ # internal use only for CI # get latest MR information by source branch # -# SPDX-FileCopyrightText: 2020-2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # - import argparse import os import subprocess @@ -14,6 +13,7 @@ import typing as t from pathlib import Path from gitlab_api import Gitlab +from idf_ci_utils import IDF_PATH if t.TYPE_CHECKING: from gitlab.v4.objects import ProjectCommit, ProjectMergeRequest @@ -59,6 +59,43 @@ def get_mr_commits(source_branch: str) -> t.List['ProjectCommit']: return list(mr.commits()) +_COMPONENT_NAME_DIR_RECORDS = {} + + +def get_modified_component(filepath: str) -> t.Optional[str]: + """Return the component name if the file is in a component directory, otherwise None.""" + try: + f_path = Path(filepath).resolve().relative_to(IDF_PATH) + except ValueError: # not in IDF_PATH + return None + + # skip md files, etc. + if f_path.suffix in ['.md', '.yml']: + return None + + # skip test_apps files + if 'test_apps' in f_path.parts: + return None + + component_parent_dirs = [f_path.parts[0]] + for part in f_path.parts[1:]: + if component_parent_dirs[-1] == 'components' or component_parent_dirs[-1].endswith('common_components'): + if part not in _COMPONENT_NAME_DIR_RECORDS: + print('Found component "%s" in path "%s"' % (part, component_parent_dirs)) + _COMPONENT_NAME_DIR_RECORDS[part] = component_parent_dirs + elif _COMPONENT_NAME_DIR_RECORDS.get(part) != component_parent_dirs: + print( + 'WARNING!!! Found component "%s" in path "%s" and "%s"' + % (part, component_parent_dirs, _COMPONENT_NAME_DIR_RECORDS.get(part)) + ) + + return part + + component_parent_dirs.append(part) + + return None + + def get_mr_components( source_branch: t.Optional[str] = None, modified_files: t.Optional[t.List[str]] = None ) -> t.List[str]: @@ -70,13 +107,9 @@ def get_mr_components( modified_files = get_mr_changed_files(source_branch) for f in modified_files: - file = Path(f) - if ( - file.parts[0] == 'components' - and 'test_apps' not in file.parts - and file.parts[-1] != '.build-test-rules.yml' - ): - components.add(file.parts[1]) + modified_component = get_modified_component(f) + if modified_component: + components.add(modified_component) return list(components) From 59bd0b3166344135ce8d9f12726931e7e20e2150 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 7 May 2024 11:22:08 +0200 Subject: [PATCH 2/3] ci: update .build-test-rules.yml based on latest detected components --- examples/ethernet/.build-test-rules.yml | 7 +++---- examples/ieee802154/.build-test-rules.yml | 2 +- examples/network/.build-test-rules.yml | 2 +- examples/protocols/.build-test-rules.yml | 14 +------------- examples/system/.build-test-rules.yml | 9 +++------ examples/wifi/.build-test-rules.yml | 5 ++--- 6 files changed, 11 insertions(+), 28 deletions(-) diff --git a/examples/ethernet/.build-test-rules.yml b/examples/ethernet/.build-test-rules.yml index 35a00eebdb..e52a806732 100644 --- a/examples/ethernet/.build-test-rules.yml +++ b/examples/ethernet/.build-test-rules.yml @@ -49,7 +49,6 @@ examples/ethernet/iperf: - console - esp_driver_gpio - esp_driver_spi - depends_filepatterns: - - examples/common_components/protocol_examples_common/**/* - - examples/system/console/advanced/components/cmd_system/**/* - - examples/ethernet/basic/components/ethernet_init/**/* + - cmd_system + - ethernet_init + - protocol_examples_common diff --git a/examples/ieee802154/.build-test-rules.yml b/examples/ieee802154/.build-test-rules.yml index 0af66989a0..da1aefce19 100644 --- a/examples/ieee802154/.build-test-rules.yml +++ b/examples/ieee802154/.build-test-rules.yml @@ -5,9 +5,9 @@ - esp_coex - esp_phy - ieee802154 + - cmd_system depends_filepatterns: - examples/ieee802154/**/* - - examples/system/console/advanced/components/cmd_system/**/* examples/ieee802154/ieee802154_cli: enable: diff --git a/examples/network/.build-test-rules.yml b/examples/network/.build-test-rules.yml index 2a62378780..d479ebc53e 100644 --- a/examples/network/.build-test-rules.yml +++ b/examples/network/.build-test-rules.yml @@ -26,7 +26,6 @@ examples/network/simple_sniffer: reason: lack of runners depends_filepatterns: - tools/ci/python_packages/common_test_methods.py - - examples/common_components/protocol_examples_common/**/* - examples/protocols/**/* - examples/wifi/**/* - examples/network/simple_sniffer/**/* @@ -35,6 +34,7 @@ examples/network/simple_sniffer: depends_components: - app_update - esp_https_ota + - protocol_examples_common examples/network/sta2eth: disable: diff --git a/examples/protocols/.build-test-rules.yml b/examples/protocols/.build-test-rules.yml index 07d49e3b3b..2c0892d6ee 100644 --- a/examples/protocols/.build-test-rules.yml +++ b/examples/protocols/.build-test-rules.yml @@ -1,8 +1,6 @@ # Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps .default_dependencies: &default_dependencies - depends_filepatterns: - - examples/common_components/protocol_examples_common/**/* depends_components: - lwip - esp_netif @@ -10,6 +8,7 @@ - esp_phy - esp_eth - mbedtls + - protocol_examples_common examples/protocols/esp_http_client: <<: *default_dependencies @@ -22,7 +21,6 @@ examples/protocols/esp_http_client: disable_test: - if: IDF_TARGET not in ["esp32", "linux"] depends_filepatterns: - - examples/common_components/protocol_examples_common/**/* - components/esp_http_client/**/* examples/protocols/esp_local_ctrl: @@ -35,7 +33,6 @@ examples/protocols/esp_local_ctrl: - if: IDF_TARGET not in ["esp32", "esp32c3", "esp32s3"] reason: only test on these targets depends_filepatterns: - - examples/common_components/protocol_examples_common/**/* - components/protocomm/**/* - tools/esp_prov/**/* @@ -59,7 +56,6 @@ examples/protocols/http_server: - if: IDF_TARGET not in ["esp32", "esp32c3", "esp32s3"] reason: only test on these targets depends_filepatterns: - - examples/common_components/protocol_examples_common/**/* - components/esp_http_server/**/* examples/protocols/http_server/captive_portal: @@ -72,7 +68,6 @@ examples/protocols/http_server/captive_portal: - if: IDF_TARGET != "esp32" reason: only test on esp32 depends_filepatterns: - - examples/common_components/protocol_examples_common/**/* - components/esp_http_server/**/* examples/protocols/http_server/restful_server: @@ -82,7 +77,6 @@ examples/protocols/http_server/restful_server: temporary: true reason: not supported yet # TODO: [ESP32C5] IDF-8697 depends_filepatterns: - - examples/common_components/protocol_examples_common/**/* - components/esp_http_server/**/* examples/protocols/http_server/ws_echo_server: @@ -95,7 +89,6 @@ examples/protocols/http_server/ws_echo_server: - if: IDF_TARGET != "esp32" reason: only test on esp32 depends_filepatterns: - - examples/common_components/protocol_examples_common/**/* - components/esp_http_server/**/* examples/protocols/https_mbedtls: @@ -130,7 +123,6 @@ examples/protocols/https_server/simple: - if: IDF_TARGET not in ["esp32", "esp32c3", "esp32s3"] reason: only test on these targets depends_filepatterns: - - examples/common_components/protocol_examples_common/**/* - components/esp_https_server/**/* - components/esp_tls/**/* @@ -144,7 +136,6 @@ examples/protocols/https_server/wss_server: - if: IDF_TARGET != "esp32" reason: only test on esp32 depends_filepatterns: - - examples/common_components/protocol_examples_common/**/* - components/esp_https_server/**/* examples/protocols/https_x509_bundle: @@ -157,7 +148,6 @@ examples/protocols/https_x509_bundle: - if: IDF_TARGET != "esp32" reason: only test on esp32 depends_filepatterns: - - examples/common_components/protocol_examples_common/**/* - components/esp_tls/**/* examples/protocols/icmp_echo: @@ -186,7 +176,6 @@ examples/protocols/modbus: temporary: true reason: not supported yet # TODO: [ESP32C5] IDF-8722, IDF-8697 depends_filepatterns: - - examples/common_components/protocol_examples_common/**/* - examples/protocols/modbus/mb_example_common/**/* examples/protocols/mqtt: @@ -196,7 +185,6 @@ examples/protocols/mqtt: temporary: true reason: not supported on c5 # TODO: [ESP32C5] IDF-8697 depends_filepatterns: - - examples/common_components/protocol_examples_common/**/* - components/mqtt/**/* examples/protocols/mqtt/custom_outbox: diff --git a/examples/system/.build-test-rules.yml b/examples/system/.build-test-rules.yml index c9c8fa36b1..06fc045042 100644 --- a/examples/system/.build-test-rules.yml +++ b/examples/system/.build-test-rules.yml @@ -148,8 +148,7 @@ examples/system/ota/advanced_https_ota: - esp_wifi - esp_phy - bt - depends_filepatterns: - - examples/common_components/protocol_examples_common/**/* + - protocol_examples_common examples/system/ota/native_ota_example: disable: @@ -166,8 +165,7 @@ examples/system/ota/native_ota_example: - esp_eth - esp_driver_gpio - bootloader_support - depends_filepatterns: - - examples/common_components/protocol_examples_common/**/* + - protocol_examples_common examples/system/ota/otatool: disable: @@ -191,8 +189,7 @@ examples/system/ota/pre_encrypted_ota: - esp_eth - mbedtls - bootloader_support - depends_filepatterns: - - examples/common_components/protocol_examples_common/**/* + - protocol_examples_common examples/system/ota/simple_ota_example: disable: diff --git a/examples/wifi/.build-test-rules.yml b/examples/wifi/.build-test-rules.yml index 1a6e8bc6d6..1c6124963d 100644 --- a/examples/wifi/.build-test-rules.yml +++ b/examples/wifi/.build-test-rules.yml @@ -12,8 +12,7 @@ - nvs_flash - console - esp_pm - depends_filepatterns: - - examples/common_components/protocol_examples_common/**/* + - protocol_examples_common examples/wifi: <<: *wifi_depends_default @@ -52,9 +51,9 @@ examples/wifi/iperf: - esp_event - esp_coex - wpa_supplicant + - cmd_system depends_filepatterns: - examples/common_components/iperf/**/* - - examples/system/console/advanced/components/cmd_system/**/* examples/wifi/itwt: <<: *wifi_depends_default From 7e1fc695ededd82da3c5e7a6576c3e49033b3836 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 7 May 2024 11:37:34 +0200 Subject: [PATCH 3/3] ci: optimize wifi .build-test-rules.yml --- examples/wifi/.build-test-rules.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/examples/wifi/.build-test-rules.yml b/examples/wifi/.build-test-rules.yml index 1c6124963d..0c91cd8448 100644 --- a/examples/wifi/.build-test-rules.yml +++ b/examples/wifi/.build-test-rules.yml @@ -51,9 +51,6 @@ examples/wifi/iperf: - esp_event - esp_coex - wpa_supplicant - - cmd_system - depends_filepatterns: - - examples/common_components/iperf/**/* examples/wifi/itwt: <<: *wifi_depends_default