Merge branch 'ci/consider_more_components_dirs' into 'master'

ci: improve get_mr_components, take more components into consideration

See merge request espressif/esp-idf!30673

examples/ethernet/.build-test-rules.yml

@@ -49,7 +49,6 @@ examples/ethernet/iperf:
     - console
     - esp_driver_gpio
     - esp_driver_spi
-  depends_filepatterns:
+    - cmd_system
-    - examples/common_components/protocol_examples_common/**/*
+    - ethernet_init
-    - examples/system/console/advanced/components/cmd_system/**/*
+    - protocol_examples_common
-    - examples/ethernet/basic/components/ethernet_init/**/*

examples/ieee802154/.build-test-rules.yml

@@ -5,9 +5,9 @@
     - esp_coex
     - esp_phy
     - ieee802154
+    - cmd_system
   depends_filepatterns:
     - examples/ieee802154/**/*
-    - examples/system/console/advanced/components/cmd_system/**/*
 
 examples/ieee802154/ieee802154_cli:
   enable:

examples/network/.build-test-rules.yml

@@ -26,7 +26,6 @@ examples/network/simple_sniffer:
       reason: lack of runners
   depends_filepatterns:
     - tools/ci/python_packages/common_test_methods.py
-    - examples/common_components/protocol_examples_common/**/*
     - examples/protocols/**/*
     - examples/wifi/**/*
     - examples/network/simple_sniffer/**/*
@@ -35,6 +34,7 @@ examples/network/simple_sniffer:
   depends_components:
     - app_update
     - esp_https_ota
+    - protocol_examples_common
 
 examples/network/sta2eth:
   disable:

examples/protocols/.build-test-rules.yml

@@ -1,8 +1,6 @@
 # Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
 
 .default_dependencies: &default_dependencies
-  depends_filepatterns:
-    - examples/common_components/protocol_examples_common/**/*
   depends_components:
     - lwip
     - esp_netif
@@ -10,6 +8,7 @@
     - esp_phy
     - esp_eth
     - mbedtls
+    - protocol_examples_common
 
 examples/protocols/esp_http_client:
   <<: *default_dependencies
@@ -22,7 +21,6 @@ examples/protocols/esp_http_client:
   disable_test:
     - if: IDF_TARGET not in ["esp32", "linux"]
   depends_filepatterns:
-    - examples/common_components/protocol_examples_common/**/*
     - components/esp_http_client/**/*
 
 examples/protocols/esp_local_ctrl:
@@ -35,7 +33,6 @@ examples/protocols/esp_local_ctrl:
     - if: IDF_TARGET not in ["esp32", "esp32c3", "esp32s3"]
       reason: only test on these targets
   depends_filepatterns:
-    - examples/common_components/protocol_examples_common/**/*
     - components/protocomm/**/*
     - tools/esp_prov/**/*
 
@@ -59,7 +56,6 @@ examples/protocols/http_server:
     - if: IDF_TARGET not in ["esp32", "esp32c3", "esp32s3"]
       reason: only test on these targets
   depends_filepatterns:
-    - examples/common_components/protocol_examples_common/**/*
     - components/esp_http_server/**/*
 
 examples/protocols/http_server/captive_portal:
@@ -72,7 +68,6 @@ examples/protocols/http_server/captive_portal:
     - if: IDF_TARGET != "esp32"
       reason: only test on esp32
   depends_filepatterns:
-    - examples/common_components/protocol_examples_common/**/*
     - components/esp_http_server/**/*
 
 examples/protocols/http_server/restful_server:
@@ -82,7 +77,6 @@ examples/protocols/http_server/restful_server:
       temporary: true
       reason: not supported yet  # TODO: [ESP32C5] IDF-8697
   depends_filepatterns:
-    - examples/common_components/protocol_examples_common/**/*
     - components/esp_http_server/**/*
 
 examples/protocols/http_server/ws_echo_server:
@@ -95,7 +89,6 @@ examples/protocols/http_server/ws_echo_server:
     - if: IDF_TARGET != "esp32"
       reason: only test on esp32
   depends_filepatterns:
-    - examples/common_components/protocol_examples_common/**/*
     - components/esp_http_server/**/*
 
 examples/protocols/https_mbedtls:
@@ -130,7 +123,6 @@ examples/protocols/https_server/simple:
     - if: IDF_TARGET not in ["esp32", "esp32c3", "esp32s3"]
       reason: only test on these targets
   depends_filepatterns:
-    - examples/common_components/protocol_examples_common/**/*
     - components/esp_https_server/**/*
     - components/esp_tls/**/*
 
@@ -144,7 +136,6 @@ examples/protocols/https_server/wss_server:
     - if: IDF_TARGET != "esp32"
       reason: only test on esp32
   depends_filepatterns:
-    - examples/common_components/protocol_examples_common/**/*
     - components/esp_https_server/**/*
 
 examples/protocols/https_x509_bundle:
@@ -157,7 +148,6 @@ examples/protocols/https_x509_bundle:
     - if: IDF_TARGET != "esp32"
       reason: only test on esp32
   depends_filepatterns:
-    - examples/common_components/protocol_examples_common/**/*
     - components/esp_tls/**/*
 
 examples/protocols/icmp_echo:
@@ -186,7 +176,6 @@ examples/protocols/modbus:
       temporary: true
       reason: not supported yet  # TODO: [ESP32C5] IDF-8722, IDF-8697
   depends_filepatterns:
-    - examples/common_components/protocol_examples_common/**/*
     - examples/protocols/modbus/mb_example_common/**/*
 
 examples/protocols/mqtt:
@@ -196,7 +185,6 @@ examples/protocols/mqtt:
       temporary: true
       reason: not supported on c5 # TODO: [ESP32C5] IDF-8697
   depends_filepatterns:
-    - examples/common_components/protocol_examples_common/**/*
     - components/mqtt/**/*
 
 examples/protocols/mqtt/custom_outbox:

examples/system/.build-test-rules.yml

@@ -148,8 +148,7 @@ examples/system/ota/advanced_https_ota:
     - esp_wifi
     - esp_phy
     - bt
-  depends_filepatterns:
+    - protocol_examples_common
-    - examples/common_components/protocol_examples_common/**/*
 
 examples/system/ota/native_ota_example:
   disable:
@@ -166,8 +165,7 @@ examples/system/ota/native_ota_example:
     - esp_eth
     - esp_driver_gpio
     - bootloader_support
-  depends_filepatterns:
+    - protocol_examples_common
-    - examples/common_components/protocol_examples_common/**/*
 
 examples/system/ota/otatool:
   disable:
@@ -191,8 +189,7 @@ examples/system/ota/pre_encrypted_ota:
     - esp_eth
     - mbedtls
     - bootloader_support
-  depends_filepatterns:
+    - protocol_examples_common
-    - examples/common_components/protocol_examples_common/**/*
 
 examples/system/ota/simple_ota_example:
   disable:

examples/wifi/.build-test-rules.yml

@@ -12,8 +12,7 @@
     - nvs_flash
     - console
     - esp_pm
-  depends_filepatterns:
+    - protocol_examples_common
-    - examples/common_components/protocol_examples_common/**/*
 
 examples/wifi:
   <<: *wifi_depends_default
@@ -52,9 +51,6 @@ examples/wifi/iperf:
     - esp_event
     - esp_coex
     - wpa_supplicant
-  depends_filepatterns:
-    - examples/common_components/iperf/**/*
-    - examples/system/console/advanced/components/cmd_system/**/*
 
 examples/wifi/itwt:
   <<: *wifi_depends_default

tools/ci/ci_get_mr_info.py

@@ -3,10 +3,9 @@
 # internal use only for CI
 # get latest MR information by source branch
 #
-# SPDX-FileCopyrightText: 2020-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 #
-
 import argparse
 import os
 import subprocess
@@ -14,6 +13,7 @@ import typing as t
 from pathlib import Path
 
 from gitlab_api import Gitlab
+from idf_ci_utils import IDF_PATH
 
 if t.TYPE_CHECKING:
     from gitlab.v4.objects import ProjectCommit, ProjectMergeRequest
@@ -59,6 +59,43 @@ def get_mr_commits(source_branch: str) -> t.List['ProjectCommit']:
     return list(mr.commits())
 
 
+_COMPONENT_NAME_DIR_RECORDS = {}
+
+
+def get_modified_component(filepath: str) -> t.Optional[str]:
+    """Return the component name if the file is in a component directory, otherwise None."""
+    try:
+        f_path = Path(filepath).resolve().relative_to(IDF_PATH)
+    except ValueError:  # not in IDF_PATH
+        return None
+
+    # skip md files, etc.
+    if f_path.suffix in ['.md', '.yml']:
+        return None
+
+    # skip test_apps files
+    if 'test_apps' in f_path.parts:
+        return None
+
+    component_parent_dirs = [f_path.parts[0]]
+    for part in f_path.parts[1:]:
+        if component_parent_dirs[-1] == 'components' or component_parent_dirs[-1].endswith('common_components'):
+            if part not in _COMPONENT_NAME_DIR_RECORDS:
+                print('Found component "%s" in path "%s"' % (part, component_parent_dirs))
+                _COMPONENT_NAME_DIR_RECORDS[part] = component_parent_dirs
+            elif _COMPONENT_NAME_DIR_RECORDS.get(part) != component_parent_dirs:
+                print(
+                    'WARNING!!! Found component "%s" in path "%s" and "%s"'
+                    % (part, component_parent_dirs, _COMPONENT_NAME_DIR_RECORDS.get(part))
+                )
+
+            return part
+
+        component_parent_dirs.append(part)
+
+    return None
+
+
 def get_mr_components(
     source_branch: t.Optional[str] = None, modified_files: t.Optional[t.List[str]] = None
 ) -> t.List[str]:
@@ -70,13 +107,9 @@ def get_mr_components(
         modified_files = get_mr_changed_files(source_branch)
 
     for f in modified_files:
-        file = Path(f)
+        modified_component = get_modified_component(f)
-        if (
+        if modified_component:
-            file.parts[0] == 'components'
+            components.add(modified_component)
-            and 'test_apps' not in file.parts
-            and file.parts[-1] != '.build-test-rules.yml'
-        ):
-            components.add(file.parts[1])
 
     return list(components)