blackbeard420/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf synced 2025-03-09 17:19:09 -04:00
Code Issues Projects Releases Wiki Activity

Merge branch 'ci/add-python-func-and-fix-c-testcase-path-in-junit-report' into 'master'

Browse Source 
ci: add python_func attribute and fix C testcase path resolution in JUnit reports

Closes RDT-1013, IDFCI-1990, IDFCI-1964, and IDFCI-1429

See merge request espressif/esp-idf!35058
This commit is contained in:
Aleksei Apaseev 2025-02-20 10:09:52 +08:00
commit 8355b832d3
7 changed files with 105 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
tools/ci/dynamic_pipelines/models.py
View File

@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import inspect
import os
@ -8,6 +8,7 @@ from dataclasses import dataclass
from xml.etree.ElementTree import Element
import yaml
from idf_ci_utils import IDF_PATH
class Job:
@ -130,6 +131,7 @@ class TestCase:
name: str
file: str
time: float
app_path: t.Optional[str] = None
failure: t.Optional[str] = None
skipped: t.Optional[str] = None
ci_job_url: t.Optional[str] = None
@ -150,6 +152,13 @@ class TestCase:
def is_success(self) -> bool:
return not self.is_failure and not self.is_skipped
@classmethod
def _get_idf_rel_path(cls, path: str) -> str:
if path.startswith(IDF_PATH):
return os.path.relpath(path, IDF_PATH)
else:
return path
@classmethod
def from_test_case_node(cls, node: Element) -> t.Optional['TestCase']:
if 'name' not in node.attrib:
@ -163,6 +172,9 @@ class TestCase:
kwargs = {
'name': node.attrib['name'],
'file': node.attrib.get('file'),
'app_path': '|'.join(
cls._get_idf_rel_path(path) for path in node.attrib.get('app_path', 'unknown').split('|')
),
'time': float(node.attrib.get('time') or 0),
'ci_job_url': node.attrib.get('ci_job_url') or 'Not found',
'ci_dashboard_url': f'{grafana_base_url}?{encoded_params}',

20
tools/ci/dynamic_pipelines/report.py
View File

@ -788,7 +788,7 @@ class TargetTestReportGenerator(ReportGenerator):
items=failed_test_cases_cur_branch,
headers=[
'Test Case',
'Test Script File Path',
'Test App Path',
'Failure Reason',
f'Failures on your branch (40 latest testcases)',
'Dut Log URL',
@ -796,7 +796,7 @@ class TargetTestReportGenerator(ReportGenerator):
'Job URL',
'Grafana URL',
],
row_attrs=['name', 'file', 'failure', 'dut_log_url', 'ci_job_url', 'ci_dashboard_url'],
row_attrs=['name', 'app_path', 'failure', 'dut_log_url', 'ci_job_url', 'ci_dashboard_url'],
value_functions=[
(
'Failures on your branch (40 latest testcases)',
@ -813,7 +813,7 @@ class TargetTestReportGenerator(ReportGenerator):
items=failed_test_cases_other_branch,
headers=[
'Test Case',
'Test Script File Path',
'Test App Path',
'Failure Reason',
'Cases that failed in other branches as well (40 latest testcases)',
'Dut Log URL',
@ -821,7 +821,7 @@ class TargetTestReportGenerator(ReportGenerator):
'Job URL',
'Grafana URL',
],
row_attrs=['name', 'file', 'failure', 'dut_log_url', 'ci_job_url', 'ci_dashboard_url'],
row_attrs=['name', 'app_path', 'failure', 'dut_log_url', 'ci_job_url', 'ci_dashboard_url'],
value_functions=[
(
'Cases that failed in other branches as well (40 latest testcases)',
@ -836,8 +836,8 @@ class TargetTestReportGenerator(ReportGenerator):
known_failures_cases_table_section = self.create_table_section(
title=self.report_titles_map['failed_known'],
items=known_failures,
headers=['Test Case', 'Test Script File Path', 'Failure Reason', 'Job URL', 'Grafana URL'],
row_attrs=['name', 'file', 'failure', 'ci_job_url', 'ci_dashboard_url'],
headers=['Test Case', 'Test App Path', 'Failure Reason', 'Job URL', 'Grafana URL'],
row_attrs=['name', 'app_path', 'failure', 'ci_job_url', 'ci_dashboard_url'],
)
failed_cases_report_url = self.write_report_to_file(
self.generate_html_report(
@ -870,8 +870,8 @@ class TargetTestReportGenerator(ReportGenerator):
skipped_cases_table_section = self.create_table_section(
title=self.report_titles_map['skipped'],
items=skipped_test_cases,
headers=['Test Case', 'Test Script File Path', 'Skipped Reason', 'Grafana URL'],
row_attrs=['name', 'file', 'skipped', 'ci_dashboard_url'],
headers=['Test Case', 'Test App Path', 'Skipped Reason', 'Grafana URL'],
row_attrs=['name', 'app_path', 'skipped', 'ci_dashboard_url'],
)
skipped_cases_report_url = self.write_report_to_file(
self.generate_html_report(''.join(skipped_cases_table_section)),
@ -892,8 +892,8 @@ class TargetTestReportGenerator(ReportGenerator):
succeeded_cases_table_section = self.create_table_section(
title=self.report_titles_map['succeeded'],
items=succeeded_test_cases,
headers=['Test Case', 'Test Script File Path', 'Job URL', 'Grafana URL'],
row_attrs=['name', 'file', 'ci_job_url', 'ci_dashboard_url'],
headers=['Test Case', 'Test App Path', 'Job URL', 'Grafana URL'],
row_attrs=['name', 'app_path', 'ci_job_url', 'ci_dashboard_url'],
)
succeeded_cases_report_url = self.write_report_to_file(
self.generate_html_report(''.join(succeeded_cases_table_section)),

2
tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml
View File

@ -56,7 +56,7 @@
TARGET_SELECTOR: ""
ENV_MARKERS: ""
INSTALL_EXTRA_TOOLS: "xtensa-esp-elf-gdb riscv32-esp-elf-gdb openocd-esp32 esp-rom-elfs"
PYTEST_EXTRA_FLAGS: "--dev-passwd ${ETHERNET_TEST_PASSWORD} --dev-user ${ETHERNET_TEST_USER} --capture=fd --verbosity=0"
PYTEST_EXTRA_FLAGS: "--dev-passwd ${ETHERNET_TEST_PASSWORD} --dev-user ${ETHERNET_TEST_USER} --capture=fd --verbosity=0 --unity-test-report-mode merge"
cache:
# Usually do not need submodule-cache in target_test
- key: pip-cache-${LATEST_GIT_TAG}

34
tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/XUNIT_REPORT.xml
View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<testsuites>
<testsuite errors="2" failures="0" hostname="FA002598" name="pytest" skipped="0" tests="2" time="22.981" timestamp="2024-05-17T17:51:26.669364">
<testcase classname="components.driver.test_apps.i2c_test_apps.pytest_i2c" file="components/driver/test_apps/i2c_test_apps/pytest_i2c.py" line="21" name="('esp32h2', 'esp32h2').('defaults', 'defaults').test_i2c_multi_device" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/('esp32h2', 'esp32h2').('defaults', 'defaults').test_i2c_multi_device/dut.txt" time="11.910">
<testcase classname="components.driver.test_apps.i2c_test_apps.pytest_i2c" app_path="components/driver/test_apps/i2c_test_apps" file="components/driver/test_apps/i2c_test_apps/pytest_i2c.py" line="21" name="('esp32h2', 'esp32h2').('defaults', 'defaults').test_i2c_multi_device" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/('esp32h2', 'esp32h2').('defaults', 'defaults').test_i2c_multi_device/dut.txt" time="11.910">
<error message="failed on setup with &quot;EOFError&quot;">conftest.py:74: in case_tester
yield CaseTester(dut, **kwargs)
tools/ci/idf_unity_tester.py:202: in __init__
@ -18,7 +18,7 @@ tools/ci/idf_unity_tester.py:202: in __init__
raise EOFError
E EOFError</error>
</testcase>
<testcase classname="components.driver.test_apps.i2s_test_apps.i2s_multi_dev.pytest_i2s_multi_dev" file="components/driver/test_apps/i2s_test_apps/i2s_multi_dev/pytest_i2s_multi_dev.py" line="5" name="('esp32h2', 'esp32h2').('default', 'default').test_i2s_multi_dev" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/('esp32h2', 'esp32h2').('default', 'default').test_i2s_multi_dev/dut.txt" time="11.071">
<testcase classname="components.driver.test_apps.i2s_test_apps.i2s_multi_dev.pytest_i2s_multi_dev" app_path="components/driver/test_apps/i2s_test_apps/i2s_multi_dev" file="components/driver/test_apps/i2s_test_apps/i2s_multi_dev/pytest_i2s_multi_dev.py" line="5" name="('esp32h2', 'esp32h2').('default', 'default').test_i2s_multi_dev" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/('esp32h2', 'esp32h2').('default', 'default').test_i2s_multi_dev/dut.txt" time="11.071">
<error message="failed on setup with &quot;EOFError&quot;">conftest.py:74: in case_tester
yield CaseTester(dut, **kwargs)
tools/ci/idf_unity_tester.py:202: in __init__
@ -37,9 +37,9 @@ E EOFError</error>
</testcase>
</testsuite>
<testsuite errors="0" failures="1" hostname="GX64-C2-SH-1-ITS1N4" name="pytest" skipped="0" tests="3" time="101.163" timestamp="2024-05-17T17:52:04.061589">
<testcase classname="components.vfs.test_apps.pytest_vfs" file="components/vfs/test_apps/pytest_vfs.py" line="7" name="esp32c2.default.test_vfs_default" dut_log_url="https://https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c2.default.test_vfs_default/dut.txt" time="30.044"/>
<testcase classname="components.vfs.test_apps.pytest_vfs" file="components/vfs/test_apps/pytest_vfs.py" line="7" name="esp32c2.iram.test_vfs_default" dut_log_url="https://https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c2.iram.test_vfs_default/dut.txt" time="28.323"/>
<testcase classname="components.wpa_supplicant.test_apps.pytest_wpa_supplicant_ut" file="components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py" line="8" name="esp32c2.default.test_wpa_supplicant_ut" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c2.default.test_wpa_supplicant_ut/dut.txt" time="42.796">
<testcase classname="components.vfs.test_apps.pytest_vfs" app_path="components/vfs/test_apps" file="components/vfs/test_apps/pytest_vfs.py" line="7" name="esp32c2.default.test_vfs_default" dut_log_url="https://https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c2.default.test_vfs_default/dut.txt" time="30.044"/>
<testcase classname="components.vfs.test_apps.pytest_vfs" app_path="components/vfs/test_apps" file="components/vfs/test_apps/pytest_vfs.py" line="7" name="esp32c2.iram.test_vfs_default" dut_log_url="https://https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c2.iram.test_vfs_default/dut.txt" time="28.323"/>
<testcase classname="components.wpa_supplicant.test_apps.pytest_wpa_supplicant_ut" app_path="components/wpa_supplicant/test_apps" file="components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py" line="8" name="esp32c2.default.test_wpa_supplicant_ut" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c2.default.test_wpa_supplicant_ut/dut.txt" time="42.796">
<failure message="AssertionError: Unity test failed">/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/plugin.py:1272: in pytest_runtest_call
self._raise_dut_failed_cases_if_exists(duts) # type: ignore
/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/plugin.py:1207: in _raise_dut_failed_cases_if_exists
@ -48,19 +48,19 @@ E AssertionError: Unity test failed</failure>
</testcase>
</testsuite>
<testsuite errors="0" failures="0" hostname="runner-zmdq2hnf-project-103-concurrent-3" name="pytest" skipped="1" tests="8" time="123.596" timestamp="2024-05-17T03:04:11.412971">
<testcase classname="test_common" file="test_common.py" line="114" name="test_python_interpreter_unix" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/test_python_interpreter_unix/dut.txt" time="7.523"/>
<testcase classname="test_common" file="test_common.py" line="133" name="test_python_interpreter_win" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/test_python_interpreter_win/dut.txt" time="0.000">
<testcase classname="test_common" app_path="tools" file="test_common.py" line="114" name="test_python_interpreter_unix" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/test_python_interpreter_unix/dut.txt" time="7.523"/>
<testcase classname="test_common" app_path="tools" file="test_common.py" line="133" name="test_python_interpreter_win" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/test_python_interpreter_win/dut.txt" time="0.000">
<skipped message="Linux does not support executing .exe files" type="pytest.skip">/builds/espressif/esp-idf/tools/test_build_system/test_common.py:134: Linux does not support executing .exe files</skipped>
</testcase>
<testcase classname="test_common" file="test_common.py" line="147" name="test_invoke_confserver" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/test_invoke_confserver/dut.txt" time="10.179"/>
<testcase classname="test_common" file="test_common.py" line="153" name="test_ccache_used_to_build" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/test_ccache_used_to_build/dut.txt" time="23.713"/>
<testcase classname="test_common" file="test_common.py" line="171" name="test_toolchain_prefix_in_description_file" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/test_toolchain_prefix_in_description_file/dut.txt" time="8.390"/>
<testcase classname="test_common" file="test_common.py" line="178" name="test_subcommands_with_options" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/test_subcommands_with_options/dut.txt" time="28.118"/>
<testcase classname="test_common" file="test_common.py" line="194" name="test_fallback_to_build_system_target" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/test_fallback_to_build_system_target/dut.txt" time="11.687"/>
<testcase classname="test_common" file="test_common.py" line="203" name="test_create_component_project" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/test_create_component_project/dut.txt" time="33.986"/>
<testcase classname="test_common" app_path="tools" file="test_common.py" line="147" name="test_invoke_confserver" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/test_invoke_confserver/dut.txt" time="10.179"/>
<testcase classname="test_common" app_path="tools" file="test_common.py" line="153" name="test_ccache_used_to_build" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/test_ccache_used_to_build/dut.txt" time="23.713"/>
<testcase classname="test_common" app_path="tools" file="test_common.py" line="171" name="test_toolchain_prefix_in_description_file" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/test_toolchain_prefix_in_description_file/dut.txt" time="8.390"/>
<testcase classname="test_common" app_path="tools" file="test_common.py" line="178" name="test_subcommands_with_options" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/test_subcommands_with_options/dut.txt" time="28.118"/>
<testcase classname="test_common" app_path="tools" file="test_common.py" line="194" name="test_fallback_to_build_system_target" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/test_fallback_to_build_system_target/dut.txt" time="11.687"/>
<testcase classname="test_common" app_path="tools" file="test_common.py" line="203" name="test_create_component_project" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/test_create_component_project/dut.txt" time="33.986"/>
</testsuite>
<testsuite errors="0" failures="4" hostname="FA002285" name="pytest" skipped="0" tests="4" time="231.048" timestamp="2024-05-17T17:50:02.291973">
<testcase classname="components.esp_timer.test_apps.pytest_esp_timer_ut" file="components/esp_timer/test_apps/pytest_esp_timer_ut.py" line="20" name="esp32c3.release.test_esp_timer" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.release.test_esp_timer/dut.txt" time="39.686">
<testcase classname="components.esp_timer.test_apps.pytest_esp_timer_ut" app_path="components/esp_timer/test_apps" file="components/esp_timer/test_apps/pytest_esp_timer_ut.py" line="20" name="esp32c3.release.test_esp_timer" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.release.test_esp_timer/dut.txt" time="39.686">
<failure message="pexpect.exceptions.TIMEOUT: Not found &quot;Press ENTER to see the list of tests&quot;
Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710
Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt">/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper
@ -95,7 +95,7 @@ E pexpect.exceptions.TIMEOUT: Not found &quot;Press ENTER to see the list of t
E Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710
E Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt</failure>
</testcase>
<testcase classname="components.wear_levelling.test_apps.pytest_wear_levelling" file="components/wear_levelling/test_apps/pytest_wear_levelling.py" line="7" name="esp32c3.512safe.test_wear_levelling" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.512safe.test_wear_levelling/dut.txt" time="69.850">
<testcase classname="components.wear_levelling.test_apps.pytest_wear_levelling" app_path="components/wear_levelling/test_apps" file="components/wear_levelling/test_apps/pytest_wear_levelling.py" line="7" name="esp32c3.512safe.test_wear_levelling" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.512safe.test_wear_levelling/dut.txt" time="69.850">
<failure message="pexpect.exceptions.TIMEOUT: Not found &quot;re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P&lt;result&gt;OK|FAIL)', re.MULTILINE)&quot;
Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes)
Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt">/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper
@ -128,7 +128,7 @@ E pexpect.exceptions.TIMEOUT: Not found &quot;re.compile(b'^[-]+\\s*(\\d+) Tes
E Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes)
E Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt</failure>
</testcase>
<testcase classname="components.wear_levelling.test_apps.pytest_wear_levelling" file="components/wear_levelling/test_apps/pytest_wear_levelling.py" line="7" name="esp32c3.release.test_wear_levelling" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.release.test_wear_levelling/dut.txt" time="70.304">
<testcase classname="components.wear_levelling.test_apps.pytest_wear_levelling" app_path="components/wear_levelling/test_apps" file="components/wear_levelling/test_apps/pytest_wear_levelling.py" line="7" name="esp32c3.release.test_wear_levelling" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.release.test_wear_levelling/dut.txt" time="70.304">
<failure message="pexpect.exceptions.TIMEOUT: Not found &quot;re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P&lt;result&gt;OK|FAIL)', re.MULTILINE)&quot;
Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 24528 bytes)
Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_wear_levelling/dut.txt">/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper
@ -161,7 +161,7 @@ E pexpect.exceptions.TIMEOUT: Not found &quot;re.compile(b'^[-]+\\s*(\\d+) Tes
E Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 24528 bytes)
E Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_wear_levelling/dut.txt</failure>
</testcase>
<testcase classname="components.wpa_supplicant.test_apps.pytest_wpa_supplicant_ut" file="components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py" line="8" name="esp32c3.default.test_wpa_supplicant_ut" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.default.test_wpa_supplicant_ut/dut.txt" time="51.208">
<testcase classname="components.wpa_supplicant.test_apps.pytest_wpa_supplicant_ut" app_path="components/wpa_supplicant/test_apps" file="components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py" line="8" name="esp32c3.default.test_wpa_supplicant_ut" dut_log_url="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.default.test_wpa_supplicant_ut/dut.txt" time="51.208">
<failure message="pexpect.exceptions.TIMEOUT: Not found &quot;Press ENTER to see the list of tests&quot;
Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0
Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txt">/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper

50
tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html
View File

@ -59,7 +59,7 @@
<thead>
<tr>
<th>Test Case</th>
<th>Test Script File Path</th>
<th>Test App Path</th>
<th>Failure Reason</th>
<th>Cases that failed in other branches as well (40 latest testcases)</th>
<th>Dut Log URL</th>
@ -71,7 +71,7 @@
<tbody>
<tr>
<td>('esp32h2', 'esp32h2').('defaults', 'defaults').test_i2c_multi_device</td>
<td>components/driver/test_apps/i2c_test_apps/pytest_i2c.py</td>
<td>components/driver/test_apps/i2c_test_apps</td>
<td>failed on setup with "EOFError"</td>
<td>0 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/('esp32h2', 'esp32h2').('defaults', 'defaults').test_i2c_multi_device/dut.txt">link</a></td>
@ -81,7 +81,7 @@
</tr>
<tr>
<td>esp32c3.release.test_esp_timer</td>
<td>components/esp_timer/test_apps/pytest_esp_timer_ut.py</td>
<td>components/esp_timer/test_apps</td>
<td>pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt</td>
<td>0 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.release.test_esp_timer/dut.txt">link</a></td>
@ -91,7 +91,7 @@
</tr>
<tr>
<td>esp32c3.default.test_wpa_supplicant_ut</td>
<td>components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py</td>
<td>components/wpa_supplicant/test_apps</td>
<td>pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txt</td>
<td>0 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.default.test_wpa_supplicant_ut/dut.txt">link</a></td>
@ -101,7 +101,7 @@
</tr>
<tr>
<td>('esp32h2', 'esp32h2').('default', 'default').test_i2s_multi_dev</td>
<td>components/driver/test_apps/i2s_test_apps/i2s_multi_dev/pytest_i2s_multi_dev.py</td>
<td>components/driver/test_apps/i2s_test_apps/i2s_multi_dev</td>
<td>failed on setup with "EOFError"</td>
<td>3 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/('esp32h2', 'esp32h2').('default', 'default').test_i2s_multi_dev/dut.txt">link</a></td>
@ -111,7 +111,7 @@
</tr>
<tr>
<td>esp32c2.default.test_wpa_supplicant_ut</td>
<td>components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py</td>
<td>components/wpa_supplicant/test_apps</td>
<td>AssertionError: Unity test failed</td>
<td>3 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c2.default.test_wpa_supplicant_ut/dut.txt">link</a></td>
@ -121,7 +121,7 @@
</tr>
<tr>
<td>esp32c3.512safe.test_wear_levelling</td>
<td>components/wear_levelling/test_apps/pytest_wear_levelling.py</td>
<td>components/wear_levelling/test_apps</td>
<td>pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt</td>
<td>3 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.512safe.test_wear_levelling/dut.txt">link</a></td>
@ -131,7 +131,7 @@
</tr>
<tr>
<td>esp32c3.release.test_wear_levelling</td>
<td>components/wear_levelling/test_apps/pytest_wear_levelling.py</td>
<td>components/wear_levelling/test_apps</td>
<td>pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 24528 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_wear_levelling/dut.txt</td>
<td>3 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.release.test_wear_levelling/dut.txt">link</a></td>
@ -145,7 +145,7 @@
<thead>
<tr>
<th>Test Case</th>
<th>Test Script File Path</th>
<th>Test App Path</th>
<th>Failure Reason</th>
<th>Job URL</th>
<th>Grafana URL</th>
@ -154,28 +154,28 @@
<tbody>
<tr>
<td>esp32c2.default.test_wpa_supplicant_ut</td>
<td>components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py</td>
<td>components/wpa_supplicant/test_apps</td>
<td>AssertionError: Unity test failed</td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c2.default.test_wpa_supplicant_ut">link</a></td>
</tr>
<tr>
<td>esp32c3.release.test_esp_timer</td>
<td>components/esp_timer/test_apps/pytest_esp_timer_ut.py</td>
<td>components/esp_timer/test_apps</td>
<td>pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt</td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c3.release.test_esp_timer">link</a></td>
</tr>
<tr>
<td>esp32c3.512safe.test_wear_levelling</td>
<td>components/wear_levelling/test_apps/pytest_wear_levelling.py</td>
<td>components/wear_levelling/test_apps</td>
<td>pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt</td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c3.512safe.test_wear_levelling">link</a></td>
</tr>
<tr>
<td>esp32c3.default.test_wpa_supplicant_ut</td>
<td>components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py</td>
<td>components/wpa_supplicant/test_apps</td>
<td>pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txt</td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c3.default.test_wpa_supplicant_ut">link</a></td>
@ -186,7 +186,7 @@
<thead>
<tr>
<th>Test Case</th>
<th>Test Script File Path</th>
<th>Test App Path</th>
<th>Skipped Reason</th>
<th>Grafana URL</th>
</tr>
@ -194,7 +194,7 @@
<tbody>
<tr>
<td>test_python_interpreter_win</td>
<td>test_common.py</td>
<td>tools</td>
<td>Linux does not support executing .exe files</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=test_python_interpreter_win">link</a></td>
</tr>
@ -204,7 +204,7 @@
<thead>
<tr>
<th>Test Case</th>
<th>Test Script File Path</th>
<th>Test App Path</th>
<th>Job URL</th>
<th>Grafana URL</th>
</tr>
@ -212,55 +212,55 @@
<tbody>
<tr>
<td>esp32c2.default.test_vfs_default</td>
<td>components/vfs/test_apps/pytest_vfs.py</td>
<td>components/vfs/test_apps</td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c2.default.test_vfs_default">link</a></td>
</tr>
<tr>
<td>esp32c2.iram.test_vfs_default</td>
<td>components/vfs/test_apps/pytest_vfs.py</td>
<td>components/vfs/test_apps</td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c2.iram.test_vfs_default">link</a></td>
</tr>
<tr>
<td>test_python_interpreter_unix</td>
<td>test_common.py</td>
<td>tools</td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=test_python_interpreter_unix">link</a></td>
</tr>
<tr>
<td>test_invoke_confserver</td>
<td>test_common.py</td>
<td>tools</td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=test_invoke_confserver">link</a></td>
</tr>
<tr>
<td>test_ccache_used_to_build</td>
<td>test_common.py</td>
<td>tools</td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=test_ccache_used_to_build">link</a></td>
</tr>
<tr>
<td>test_toolchain_prefix_in_description_file</td>
<td>test_common.py</td>
<td>tools</td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=test_toolchain_prefix_in_description_file">link</a></td>
</tr>
<tr>
<td>test_subcommands_with_options</td>
<td>test_common.py</td>
<td>tools</td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=test_subcommands_with_options">link</a></td>
</tr>
<tr>
<td>test_fallback_to_build_system_target</td>
<td>test_common.py</td>
<td>tools</td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=test_fallback_to_build_system_target">link</a></td>
</tr>
<tr>
<td>test_create_component_project</td>
<td>test_common.py</td>
<td>tools</td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=test_create_component_project">link</a></td>
</tr>

10
tools/ci/idf_pytest/plugin.py
View File

@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import importlib
import logging
@ -34,6 +34,7 @@ from .constants import SUPPORTED_TARGETS
from .utils import comma_sep_str_to_list
from .utils import format_case_id
from .utils import merge_junit_files
from .utils import normalize_testcase_file_path
IDF_PYTEST_EMBEDDED_KEY = pytest.StashKey['IdfPytestEmbedded']()
ITEM_FAILED_CASES_KEY = pytest.StashKey[list]()
@ -365,16 +366,19 @@ class IdfPytestEmbedded:
is_qemu = item.get_closest_marker('qemu') is not None
target = item.funcargs['target']
config = item.funcargs['config']
app_path = item.funcargs.get('app_path')
for junit in junits:
xml = ET.parse(junit)
testcases = xml.findall('.//testcase')
for case in testcases:
# modify the junit files
# Use from case attrib if available, otherwise fallback to the previously defined
app_path = case.attrib.get('app_path') or app_path
new_case_name = format_case_id(target, config, case.attrib['name'], is_qemu=is_qemu)
case.attrib['name'] = new_case_name
if 'file' in case.attrib:
case.attrib['file'] = case.attrib['file'].replace('/IDF/', '') # our unity test framework
# our unity test framework
case.attrib['file'] = normalize_testcase_file_path(case.attrib['file'], app_path)
if ci_job_url := os.getenv('CI_JOB_URL'):
case.attrib['ci_job_url'] = ci_job_url

34
tools/ci/idf_pytest/utils.py
View File

@ -1,6 +1,5 @@
# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import logging
import os
import typing as t
@ -65,3 +64,34 @@ def merge_junit_files(junit_files: t.List[str], target_path: str) -> None:
def comma_sep_str_to_list(s: str) -> t.List[str]:
return [s.strip() for s in s.split(',') if s.strip()]
def normalize_testcase_file_path(file: str, app_path: t.Union[str, tuple]) -> str:
"""
Normalize file paths to a consistent format, resolving relative paths based on the `app_path`.
This function ensures that file paths are correctly resolved and normalized:
- If `app_path` is a tuple, the function will try each app path in the tuple in order and join the results with ':'.
- If `app_path` is a string, the file will be resolved using that base path.
:param file: The original file path, which can be relative, absolute.
:param app_path: The base app path used to resolve relative paths, which can be a string or tuple.
:return: A normalized file path with the IDF_PATH prefix removed if applicable.
"""
def normalize_path(file_path: str, app_path: str) -> str:
"""Helper function to normalize a single path."""
if not os.path.isabs(file_path):
resolved_path = os.path.normpath(
os.path.join(app_path, file_path.removeprefix('./').removeprefix('../'))
)
else:
resolved_path = os.path.normpath(file_path)
return resolved_path.replace(f'{os.environ.get("IDF_PATH", "")}', '').replace('/IDF/', '').lstrip('/')
if isinstance(app_path, tuple):
normalized_paths = [normalize_path(file, base_path) for base_path in app_path]
return '|'.join(normalized_paths)
return normalize_path(file, app_path)