From e18125edcf9cd2d928469cb51dbd1e8eaa18a5a0 Mon Sep 17 00:00:00 2001 From: Chen Yudong Date: Fri, 8 Sep 2023 12:45:19 +0800 Subject: [PATCH] fix(ci): optimize build rules patterns add a script to check build-components-patterns in rules.yml --- .gitlab/ci/dependencies/dependencies.yml | 3 +- .gitlab/ci/rules.yml | 320 ++++++++++---------- .gitlab/ci/target-test.yml | 2 +- .pre-commit-config.yaml | 8 + tools/ci/check_rules_components_patterns.py | 92 ++++++ tools/ci/executable-list.txt | 1 + 6 files changed, 267 insertions(+), 159 deletions(-) create mode 100755 tools/ci/check_rules_components_patterns.py diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index 42f9e65745..57656976c4 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -140,8 +140,7 @@ build:integration_test: matrix: - *target_test - *all_targets - - - bt # example_test_005 - - wifi # pytest*wifi* + - - wifi # pytest*wifi* - ethernet # pytest*ethernet* - sdio # pytest*sdio* - usb # USB Device & Host tests diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index c1820eab27..b7586016bf 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -38,7 +38,17 @@ - "tools/ci/get_supported_examples.sh" .patterns-build_components: &patterns-build_components - - "components/**/*" + # components files except "test*/" "host*/" folders + # ?? to include folders less than 4 characters + - "components/*/???[!t]*/**/*" + - "components/*/??[!s]?*/**/*" + - "components/*/???/**/*" + - "components/*/??/**/*" + - "components/*/*" + - "components/*" + # Add folders excluded by "???[!t]" and "??[!s]?" + # pre-commit: tools/ci/check_rules_components_patterns.py + - "components/bt/host/**/*" .patterns-downloadable-tools: &patterns-downloadable-tools - "tools/idf_tools.py" @@ -79,32 +89,36 @@ .patterns-component_ut: &patterns-component_ut - "tools/ci/python_packages/gitlab_api.py" - - "tools/ci/python_packages/tiny_test_fw/**/*" - - "tools/ci/python_packages/ttfw_idf/**/*" - "tools/ci/python_packages/common_test_methods.py" - "tools/test_apps/configs/sdkconfig.debug_helpers" - "components/**/*" -.patterns-target_test-i154: &patterns-target_test-i154 - - "components/esp_phy/**/*" +.patterns-component_ut-i154: &patterns-component_ut-i154 + - "components/{esp_phy,esp_coex}/???[!t]*/**/*" + - "components/{esp_phy,esp_coex}/??[!s]?*/**/*" + - "components/{esp_phy,esp_coex}/???/**/*" + - "components/{esp_phy,esp_coex}/*" - "components/ieee802154/**/*" - - "components/esp_coex/**/*" .patterns-example_test-i154: &patterns-example_test-i154 - - "components/esp_netif/**/*" - - "components/lwip/**/*" - - "components/openthread/**/*" + - "components/{esp_netif,ieee802154,lwip,openthread}/???[!t]*/**/*" + - "components/{esp_netif,ieee802154,lwip,openthread}/??[!s]?*/**/*" + - "components/{esp_netif,ieee802154,lwip,openthread}/???/**/*" + - "components/{esp_netif,ieee802154,lwip,openthread}/??/**/*" + - "components/{esp_netif,ieee802154,lwip,openthread}/*" - "examples/common_components/iperf/**/*" - "examples/openthread/**/*" .patterns-target_test-wifi: &patterns-target_test-wifi - - "components/esp_netif/**/*" - - "components/lwip/**/*" + - "components/{esp_netif,lwip,esp_phy,wpa_supplicant,esp_coex}/???[!t]*/**/*" + - "components/{esp_netif,lwip,esp_phy,wpa_supplicant,esp_coex}/??[!s]?*/**/*" + - "components/{esp_netif,lwip,esp_phy,wpa_supplicant,esp_coex}/???/**/*" + - "components/{esp_netif,lwip,esp_phy,wpa_supplicant,esp_coex}/??/**/*" + - "components/{esp_netif,lwip,esp_phy,wpa_supplicant,esp_coex}/*" + +.patterns-component_ut-wifi: &patterns-component_ut-wifi - "components/esp_wifi/**/*" - - "components/esp_phy/**/*" - - "components/wpa_supplicant/**/*" - - "components/esp_coex/**/*" .patterns-integration_test-ble: &patterns-integration_test-ble - "tools/ci/python_packages/gitlab_api.py" @@ -114,7 +128,9 @@ - "components/bt/controller/lib_esp32h2/esp32h2-bt-lib" - "components/bt/host/nimble/nimble" - "components/esp_phy/lib" - - "components/esp_coex/**/*" + - "components/esp_coex/??[!s][!t]*/**/*" + - "components/esp_coex/???/**/*" + - "components/esp_coex/*" .patterns-integration_test-wifi: &patterns-integration_test-wifi - "tools/ci/python_packages/gitlab_api.py" @@ -212,18 +228,14 @@ .patterns-danger-npm: &patterns-danger-npm - ".gitlab/dangerjs/package-lock.json" -# for jobs: example_test*flash_encryption_wifi_high_traffic +# for jobs: flash_encryption_wifi_high_traffic .patterns-example_test-ota-nightly_run: &patterns-example_test-ota-nightly_run - "examples/system/ota/**/*" - "examples/common_components/protocol_examples_common/**/*" - - "components/app_update/**/*" + - "components/app_update/include/*" + - "components/app_update/*" - "components/esp_https_ota/**/*" -# for jobs: example_test_005 -.patterns-example_test-bt: &patterns-example_test-bt - - "components/bt/host/nimble/nimble" - - "examples/bluetooth/nimble/**/*" - # for jobs: example_test*ethernet* .patterns-example_test-ethernet: &patterns-example_test-ethernet - "tools/ci/python_packages/common_test_methods.py" @@ -232,21 +244,24 @@ - "examples/system/ota/**/*" - "examples/ethernet/iperf/**/*" - "examples/network/vlan_support/**/*" - - "components/esp_eth/**/*" + - "components/esp_eth/??[!s][!t]*/**/*" + - "components/esp_eth/???/**/*" + - "components/esp_eth/*" - "components/esp_netif/esp_netif_handlers.c" -# for jobs: example_test_002, example_test*wifi* +# for jobs: example_test*wifi* .patterns-example_test-wifi: &patterns-example_test-wifi - "tools/ci/python_packages/common_test_methods.py" - "examples/common_components/protocol_examples_common/**/*" - "examples/protocols/**/*" - "examples/wifi/**/*" - "examples/network/simple_sniffer/**/*" - # components + - "components/mbedtls/port/dynamic/*" - # for cases with wifi_high_traffic marker + # ota - "examples/system/ota/**/*" - - "components/app_update/**/*" + - "components/app_update/include/*" + - "components/app_update/*" - "components/esp_https_ota/**/*" # for jobs: custom_test*wifi* @@ -263,7 +278,9 @@ - "components/driver/include/driver/sdspi*.h" - "components/driver/include/driver/sdio*.h" - "components/driver/include/driver/sdmmc*.h" - - "components/sdmmc/**/*" + - "components/sdmmc/??[!s][!t]*/**/*" + - "components/sdmmc/???/**/*" + - "components/sdmmc/*" - "examples/peripherals/sdio/**/*" # for jobs: component_ut_pytest_sdio related @@ -296,17 +313,12 @@ - "components/hal/spi_flash*.c" - "components/hal/include/hal/spi_flash*.h" -# for jobs: UT_C3_FLASH -.patterns-unit_test-flash_multi: &patterns-unit_test-flash_multi - - "components/spi_flash/**/*" - - "components/hal/spi_flash*.c" - - "components/hal/include/hal/spi_flash*.h" - # for jobs: USB host and device examples .patterns-example_test-usb: &patterns-example_test-usb - "components/hal/usb*.c" - "components/hal/esp32s*/include/hal/usb*.h" - - "components/usb/**/*" + - "components/usb/*include/**/*" + - "components/usb/*" - "examples/peripherals/usb/host/**/**/**/*" - "examples/peripherals/usb/device/**/**/*" @@ -318,16 +330,24 @@ # for jobs: *_pytest_esp32x_adc: .patterns-target_test-adc: &patterns-target_test-adc + - "components/{esp_adc,driver,hal,esp_hw_support,efuse}/???[!t]*/**/*" + - "components/{esp_adc,driver,hal,esp_hw_support,efuse}/??[!s]?*/**/*" + - "components/{esp_adc,driver,hal,esp_hw_support,efuse}/???/**/*" + - "components/{esp_adc,driver,hal,esp_hw_support,efuse}/??/**/*" + - "components/{esp_adc,driver,hal,esp_hw_support,efuse}/*" + +.patterns-component_ut-adc: &patterns-component_ut-adc - "components/esp_adc/**/*" - - "components/driver/**/*" - - "components/hal/**/*" - - "components/esp_hw_support/**/*" - - "components/efuse/**/*" + +.patterns-example_test-adc: &patterns-example_test-adc - "examples/peripherals/adc/**/*" .patterns-target_test-ecdsa: &patterns-target_test-ecdsa - - "components/hal/**/*" - - "components/efuse/**/*" + - "components/{hal,efuse}/???[!t]*/**/*" + - "components/{hal,efuse}/??[!s]?*/**/*" + - "components/{hal,efuse}/???/**/*" + - "components/{hal,efuse}/??/**/*" + - "components/{hal,efuse}/*" - "components/mbedtls/port/ecdsa/*" .patterns-component_ut-nvs_encr_hmac: &patterns-component_ut-nvs_encr_hmac @@ -335,8 +355,11 @@ - "components/nvs_sec_provider/**/*" .patterns-example_test-nvs_encr_hmac: &patterns-example_test-nvs_encr_hmac - - "components/nvs_flash/**/*" - - "components/nvs_sec_provider/**/*" + - "components/{nvs_flash,nvs_sec_provider}/???[!t]*/**/*" + - "components/{nvs_flash,nvs_sec_provider}/??[!s]?*/**/*" + - "components/{nvs_flash,nvs_sec_provider}/???/**/*" + - "components/{nvs_flash,nvs_sec_provider}/??/**/*" + - "components/{nvs_flash,nvs_sec_provider}/*" - "examples/security/nvs_encryption_hmac/**/*" ############## @@ -689,22 +712,26 @@ changes: *patterns-build_system - <<: *if-dev-push changes: *patterns-component_ut + - <<: *if-dev-push + changes: *patterns-component_ut-adc - <<: *if-dev-push changes: *patterns-component_ut-flash_multi + - <<: *if-dev-push + changes: *patterns-component_ut-i154 - <<: *if-dev-push changes: *patterns-component_ut-nvs_encr_hmac - <<: *if-dev-push changes: *patterns-component_ut-sdio - <<: *if-dev-push changes: *patterns-component_ut-usb + - <<: *if-dev-push + changes: *patterns-component_ut-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - <<: *if-dev-push changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -726,22 +753,26 @@ changes: *patterns-build_system - <<: *if-dev-push changes: *patterns-component_ut + - <<: *if-dev-push + changes: *patterns-component_ut-adc - <<: *if-dev-push changes: *patterns-component_ut-flash_multi + - <<: *if-dev-push + changes: *patterns-component_ut-i154 - <<: *if-dev-push changes: *patterns-component_ut-nvs_encr_hmac - <<: *if-dev-push changes: *patterns-component_ut-sdio - <<: *if-dev-push changes: *patterns-component_ut-usb + - <<: *if-dev-push + changes: *patterns-component_ut-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - <<: *if-dev-push changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -762,22 +793,26 @@ changes: *patterns-build_system - <<: *if-dev-push changes: *patterns-component_ut + - <<: *if-dev-push + changes: *patterns-component_ut-adc - <<: *if-dev-push changes: *patterns-component_ut-flash_multi + - <<: *if-dev-push + changes: *patterns-component_ut-i154 - <<: *if-dev-push changes: *patterns-component_ut-nvs_encr_hmac - <<: *if-dev-push changes: *patterns-component_ut-sdio - <<: *if-dev-push changes: *patterns-component_ut-usb + - <<: *if-dev-push + changes: *patterns-component_ut-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - <<: *if-dev-push changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -798,22 +833,26 @@ changes: *patterns-build_system - <<: *if-dev-push changes: *patterns-component_ut + - <<: *if-dev-push + changes: *patterns-component_ut-adc - <<: *if-dev-push changes: *patterns-component_ut-flash_multi + - <<: *if-dev-push + changes: *patterns-component_ut-i154 - <<: *if-dev-push changes: *patterns-component_ut-nvs_encr_hmac - <<: *if-dev-push changes: *patterns-component_ut-sdio - <<: *if-dev-push changes: *patterns-component_ut-usb + - <<: *if-dev-push + changes: *patterns-component_ut-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - <<: *if-dev-push changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -834,22 +873,26 @@ changes: *patterns-build_system - <<: *if-dev-push changes: *patterns-component_ut + - <<: *if-dev-push + changes: *patterns-component_ut-adc - <<: *if-dev-push changes: *patterns-component_ut-flash_multi + - <<: *if-dev-push + changes: *patterns-component_ut-i154 - <<: *if-dev-push changes: *patterns-component_ut-nvs_encr_hmac - <<: *if-dev-push changes: *patterns-component_ut-sdio - <<: *if-dev-push changes: *patterns-component_ut-usb + - <<: *if-dev-push + changes: *patterns-component_ut-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - <<: *if-dev-push changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -870,22 +913,26 @@ changes: *patterns-build_system - <<: *if-dev-push changes: *patterns-component_ut + - <<: *if-dev-push + changes: *patterns-component_ut-adc - <<: *if-dev-push changes: *patterns-component_ut-flash_multi + - <<: *if-dev-push + changes: *patterns-component_ut-i154 - <<: *if-dev-push changes: *patterns-component_ut-nvs_encr_hmac - <<: *if-dev-push changes: *patterns-component_ut-sdio - <<: *if-dev-push changes: *patterns-component_ut-usb + - <<: *if-dev-push + changes: *patterns-component_ut-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - <<: *if-dev-push changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -906,22 +953,26 @@ changes: *patterns-build_system - <<: *if-dev-push changes: *patterns-component_ut + - <<: *if-dev-push + changes: *patterns-component_ut-adc - <<: *if-dev-push changes: *patterns-component_ut-flash_multi + - <<: *if-dev-push + changes: *patterns-component_ut-i154 - <<: *if-dev-push changes: *patterns-component_ut-nvs_encr_hmac - <<: *if-dev-push changes: *patterns-component_ut-sdio - <<: *if-dev-push changes: *patterns-component_ut-usb + - <<: *if-dev-push + changes: *patterns-component_ut-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - <<: *if-dev-push changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -942,22 +993,26 @@ changes: *patterns-build_system - <<: *if-dev-push changes: *patterns-component_ut + - <<: *if-dev-push + changes: *patterns-component_ut-adc - <<: *if-dev-push changes: *patterns-component_ut-flash_multi + - <<: *if-dev-push + changes: *patterns-component_ut-i154 - <<: *if-dev-push changes: *patterns-component_ut-nvs_encr_hmac - <<: *if-dev-push changes: *patterns-component_ut-sdio - <<: *if-dev-push changes: *patterns-component_ut-usb + - <<: *if-dev-push + changes: *patterns-component_ut-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - <<: *if-dev-push changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -991,8 +1046,6 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1019,8 +1072,6 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1047,8 +1098,6 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1075,8 +1124,6 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1103,8 +1150,6 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1131,8 +1176,6 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1159,8 +1202,6 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1187,8 +1228,6 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1229,7 +1268,7 @@ - <<: *if-dev-push changes: *patterns-example_test - <<: *if-dev-push - changes: *patterns-example_test-bt + changes: *patterns-example_test-adc - <<: *if-dev-push changes: *patterns-example_test-ethernet - <<: *if-dev-push @@ -1246,8 +1285,6 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1271,7 +1308,7 @@ - <<: *if-dev-push changes: *patterns-example_test - <<: *if-dev-push - changes: *patterns-example_test-bt + changes: *patterns-example_test-adc - <<: *if-dev-push changes: *patterns-example_test-ethernet - <<: *if-dev-push @@ -1288,8 +1325,6 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1314,7 +1349,7 @@ - <<: *if-dev-push changes: *patterns-example_test - <<: *if-dev-push - changes: *patterns-example_test-bt + changes: *patterns-example_test-adc - <<: *if-dev-push changes: *patterns-example_test-ethernet - <<: *if-dev-push @@ -1331,8 +1366,6 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1356,7 +1389,7 @@ - <<: *if-dev-push changes: *patterns-example_test - <<: *if-dev-push - changes: *patterns-example_test-bt + changes: *patterns-example_test-adc - <<: *if-dev-push changes: *patterns-example_test-ethernet - <<: *if-dev-push @@ -1373,8 +1406,6 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1398,7 +1429,7 @@ - <<: *if-dev-push changes: *patterns-example_test - <<: *if-dev-push - changes: *patterns-example_test-bt + changes: *patterns-example_test-adc - <<: *if-dev-push changes: *patterns-example_test-ethernet - <<: *if-dev-push @@ -1415,8 +1446,6 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1440,7 +1469,7 @@ - <<: *if-dev-push changes: *patterns-example_test - <<: *if-dev-push - changes: *patterns-example_test-bt + changes: *patterns-example_test-adc - <<: *if-dev-push changes: *patterns-example_test-ethernet - <<: *if-dev-push @@ -1457,8 +1486,6 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1482,7 +1509,7 @@ - <<: *if-dev-push changes: *patterns-example_test - <<: *if-dev-push - changes: *patterns-example_test-bt + changes: *patterns-example_test-adc - <<: *if-dev-push changes: *patterns-example_test-ethernet - <<: *if-dev-push @@ -1499,8 +1526,6 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1524,7 +1549,7 @@ - <<: *if-dev-push changes: *patterns-example_test - <<: *if-dev-push - changes: *patterns-example_test-bt + changes: *patterns-example_test-adc - <<: *if-dev-push changes: *patterns-example_test-ethernet - <<: *if-dev-push @@ -1541,8 +1566,6 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1640,14 +1663,20 @@ changes: *patterns-build_template-app - <<: *if-dev-push changes: *patterns-component_ut + - <<: *if-dev-push + changes: *patterns-component_ut-adc - <<: *if-dev-push changes: *patterns-component_ut-flash_multi + - <<: *if-dev-push + changes: *patterns-component_ut-i154 - <<: *if-dev-push changes: *patterns-component_ut-nvs_encr_hmac - <<: *if-dev-push changes: *patterns-component_ut-sdio - <<: *if-dev-push changes: *patterns-component_ut-usb + - <<: *if-dev-push + changes: *patterns-component_ut-wifi - <<: *if-dev-push changes: *patterns-custom_test - <<: *if-dev-push @@ -1657,7 +1686,7 @@ - <<: *if-dev-push changes: *patterns-example_test - <<: *if-dev-push - changes: *patterns-example_test-bt + changes: *patterns-example_test-adc - <<: *if-dev-push changes: *patterns-example_test-ethernet - <<: *if-dev-push @@ -1678,14 +1707,10 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi - <<: *if-dev-push changes: *patterns-unit_test - - <<: *if-dev-push - changes: *patterns-unit_test-flash_multi - <<: *if-dev-push changes: *patterns-unit_test-sdio @@ -1716,14 +1741,10 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi - <<: *if-dev-push changes: *patterns-unit_test - - <<: *if-dev-push - changes: *patterns-unit_test-flash_multi - <<: *if-dev-push changes: *patterns-unit_test-sdio @@ -1747,14 +1768,10 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi - <<: *if-dev-push changes: *patterns-unit_test - - <<: *if-dev-push - changes: *patterns-unit_test-flash_multi - <<: *if-dev-push changes: *patterns-unit_test-sdio @@ -1777,14 +1794,10 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi - <<: *if-dev-push changes: *patterns-unit_test - - <<: *if-dev-push - changes: *patterns-unit_test-flash_multi - <<: *if-dev-push changes: *patterns-unit_test-sdio @@ -1807,14 +1820,10 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi - <<: *if-dev-push changes: *patterns-unit_test - - <<: *if-dev-push - changes: *patterns-unit_test-flash_multi - <<: *if-dev-push changes: *patterns-unit_test-sdio @@ -1837,14 +1846,10 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi - <<: *if-dev-push changes: *patterns-unit_test - - <<: *if-dev-push - changes: *patterns-unit_test-flash_multi - <<: *if-dev-push changes: *patterns-unit_test-sdio @@ -1867,14 +1872,10 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi - <<: *if-dev-push changes: *patterns-unit_test - - <<: *if-dev-push - changes: *patterns-unit_test-flash_multi - <<: *if-dev-push changes: *patterns-unit_test-sdio @@ -1897,14 +1898,10 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi - <<: *if-dev-push changes: *patterns-unit_test - - <<: *if-dev-push - changes: *patterns-unit_test-flash_multi - <<: *if-dev-push changes: *patterns-unit_test-sdio @@ -1927,14 +1924,10 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi - <<: *if-dev-push changes: *patterns-unit_test - - <<: *if-dev-push - changes: *patterns-unit_test-flash_multi - <<: *if-dev-push changes: *patterns-unit_test-sdio @@ -1957,14 +1950,10 @@ changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-i154 - <<: *if-dev-push changes: *patterns-target_test-wifi - <<: *if-dev-push changes: *patterns-unit_test - - <<: *if-dev-push - changes: *patterns-unit_test-flash_multi - <<: *if-dev-push changes: *patterns-unit_test-sdio @@ -2006,6 +1995,8 @@ - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32 - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-component_ut-adc - <<: *if-dev-push changes: *patterns-target_test-adc @@ -2035,19 +2026,6 @@ - <<: *if-dev-push changes: *patterns-component_ut-sdio -.rules:test:component_ut-esp32-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-target_test-wifi - .rules:test:component_ut-esp32c2: rules: - <<: *if-revert-branch @@ -2073,6 +2051,8 @@ - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32c2 - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-component_ut-adc - <<: *if-dev-push changes: *patterns-target_test-adc @@ -2086,6 +2066,8 @@ - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32c2 - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-component_ut-wifi - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -2114,6 +2096,8 @@ - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32c3 - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-component_ut-adc - <<: *if-dev-push changes: *patterns-target_test-adc @@ -2166,6 +2150,8 @@ - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32c3 - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-component_ut-wifi - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -2194,6 +2180,8 @@ - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32c6 - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-component_ut-adc - <<: *if-dev-push changes: *patterns-target_test-adc @@ -2208,7 +2196,7 @@ - <<: *if-label-component_ut_esp32c6 - <<: *if-label-target_test - <<: *if-dev-push - changes: *patterns-target_test-i154 + changes: *patterns-component_ut-i154 .rules:test:component_ut-esp32h2: rules: @@ -2235,6 +2223,8 @@ - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32h2 - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-component_ut-adc - <<: *if-dev-push changes: *patterns-target_test-adc @@ -2276,6 +2266,8 @@ - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32s2 - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-component_ut-adc - <<: *if-dev-push changes: *patterns-target_test-adc @@ -2330,6 +2322,8 @@ - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32s3 - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-component_ut-adc - <<: *if-dev-push changes: *patterns-target_test-adc @@ -2369,6 +2363,8 @@ - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32s3 - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-component_ut-wifi - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -2548,6 +2544,8 @@ - <<: *if-label-example_test - <<: *if-label-example_test_esp32 - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-adc - <<: *if-dev-push changes: *patterns-target_test-adc @@ -2626,6 +2624,8 @@ - <<: *if-label-example_test - <<: *if-label-example_test_esp32c2 - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-adc - <<: *if-dev-push changes: *patterns-target_test-adc @@ -2669,6 +2669,8 @@ - <<: *if-label-example_test - <<: *if-label-example_test_esp32c3 - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-adc - <<: *if-dev-push changes: *patterns-target_test-adc @@ -2747,6 +2749,8 @@ - <<: *if-label-example_test - <<: *if-label-example_test_esp32c6 - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-adc - <<: *if-dev-push changes: *patterns-target_test-adc @@ -2790,6 +2794,8 @@ - <<: *if-label-example_test - <<: *if-label-example_test_esp32h2 - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-adc - <<: *if-dev-push changes: *patterns-target_test-adc @@ -2818,6 +2824,8 @@ - <<: *if-label-example_test - <<: *if-label-example_test_esp32s2 - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-adc - <<: *if-dev-push changes: *patterns-target_test-adc @@ -2887,6 +2895,8 @@ - <<: *if-label-example_test - <<: *if-label-example_test_esp32s3 - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-adc - <<: *if-dev-push changes: *patterns-target_test-adc @@ -2916,8 +2926,6 @@ - <<: *if-label-target_test - <<: *if-dev-push changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-target_test-i154 .rules:test:host_test: rules: diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index c1e14dd01c..ed8f8d0e71 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -638,7 +638,7 @@ pytest_components_esp32_generic_multi_device: pytest_components_esp32_wifi_two_dut: extends: - .pytest_components_dir_template - - .rules:test:component_ut-esp32-wifi + - .rules:test:component_ut-esp32 needs: - build_pytest_components_esp32 tags: [ esp32, wifi_two_dut ] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 180745efcb..a7c9d1f512 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -127,6 +127,14 @@ repos: additional_dependencies: - PyYAML == 5.3.1 pass_filenames: false + - id: check-rules-components-patterns + name: check patterns-build_components in rules.yml + entry: tools/ci/check_rules_components_patterns.py + language: python + files: 'components/.+|.gitlab/ci/rules.yml' + additional_dependencies: + - PyYAML == 5.3.1 + pass_filenames: false - id: check-generated-soc-caps-kconfig name: Check soc caps kconfig files are generated (based on components/soc/IDF_TARGET/include/soc/soc_caps.h) entry: tools/gen_soc_caps_kconfig/gen_soc_caps_kconfig.py -d 'components/soc/*/include/soc/' 'components/esp_rom/*/' 'components/spi_flash/*/' diff --git a/tools/ci/check_rules_components_patterns.py b/tools/ci/check_rules_components_patterns.py new file mode 100755 index 0000000000..f45fece470 --- /dev/null +++ b/tools/ci/check_rules_components_patterns.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +""" +Check patterns-build_components including all components files except 'test*/' +""" + +import glob +import os +from typing import List, Set + +import yaml +from idf_ci_utils import IDF_PATH + +RULES_FILE = os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml') + + +def get_component_files_from_patterns(patterns: List[str]) -> Set[str]: + files: Set[str] = set() + for pattern in patterns: + # NOTE: version 3.10: Added the root_dir and dir_fd parameters. + files = files | set(glob.iglob(pattern, recursive=True)) + return files + + +def get_components_submodule_files(submodules: List[str]) -> Set[str]: + patterns = [f'{submodule}/**/*' for submodule in submodules] + return get_component_files_from_patterns(patterns) + + +def get_components_test_files() -> Set[str]: + patterns = [ + 'components/*/test/**/*', + 'components/*/test_apps/**/*', + 'components/*/host_test/**/*', + # other test files + 'components/efuse/test_efuse_host/**/*', + 'components/esp_coex/test_md5/**/*', + 'components/esp_gdbstub/test_gdbstub_host/**/*', + 'components/esp_system/test_eh_frame_parser/**/*', + 'components/esp_wifi/test_md5/**/*', + 'components/fatfs/test_fatfsgen/**/*', + 'components/heap/test_multi_heap_host/**/*', + 'components/nvs_flash/test_nvs_host/**/*', + 'components/partition_table/test_gen_esp32part_host/**/*', + 'components/spiffs/test_spiffsgen/**/*', + ] + return get_component_files_from_patterns(patterns) + + +def get_all_components_files() -> Set[str]: + return get_component_files_from_patterns(['components/**/*']) + + +def main() -> None: + with open(RULES_FILE, 'r') as f: + rules = yaml.load(f.read(), Loader=yaml.FullLoader) + patterns_components = [p for p in rules['.patterns-build_components'] if p.startswith('components/')] + submodules = [p for p in rules['.patterns-submodule'] if p.startswith('components/')] + # resolve components in patterns_components + patterns_components = [f'{p}/**/*' if p in submodules else p for p in patterns_components] + + # get different components files + submodule_files = get_components_submodule_files(submodules) + rules_files = get_component_files_from_patterns(patterns_components) + + # first check submodules in rules_files + if check_submodule_res := submodule_files.difference(rules_files): + # only show one file in the failure message + print(f'Found submodule should in .patterns-build_components but not: {check_submodule_res.pop()}') + exit(1) + + rules_files = rules_files.difference(submodule_files) + components_test_files = get_components_test_files().difference(submodule_files) + components_build_files = get_all_components_files().difference(submodule_files).difference(components_test_files) + + if check_test_files := components_test_files & rules_files: + # rules should not include test files + print(f'Found test files should not in .patterns-build_components: {check_test_files.pop()}') + exit(1) + if check_difference := components_build_files.difference(rules_files): + # rules should include all build files + print(f'Found components files should in .patterns-build_components but not: {check_difference.pop()}') + exit(1) + if check_difference := rules_files.difference(components_build_files): + print(f'Found components files should not in .patterns-build_components: {check_difference.pop()}') + exit(1) + + +if __name__ == '__main__': + main() diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 1509cd8bdb..a3bf0fe00e 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -60,6 +60,7 @@ tools/ci/check_idf_version.sh tools/ci/check_kconfigs.py tools/ci/check_readme_links.py tools/ci/check_requirement_files.py +tools/ci/check_rules_components_patterns.py tools/ci/check_rules_yml.py tools/ci/check_soc_struct_headers.py tools/ci/check_tools_files_patterns.py