From c03b00764447069e76bff0fd1b67ad51e3a57a58 Mon Sep 17 00:00:00 2001 From: Aleksei Apaseev Date: Wed, 20 Nov 2024 15:26:18 +0800 Subject: [PATCH] ci: fix testcase path resolution in JUnit reports. switch to the different unity test report mode. add app_path to target test report --- tools/ci/dynamic_pipelines/models.py | 14 +++++- tools/ci/dynamic_pipelines/report.py | 20 ++++---- .../templates/.dynamic_jobs.yml | 2 +- .../reports_sample_data/XUNIT_REPORT.xml | 34 ++++++------- .../expected_target_test_report.html | 50 +++++++++---------- tools/ci/idf_pytest/plugin.py | 10 ++-- tools/ci/idf_pytest/utils.py | 34 ++++++++++++- 7 files changed, 105 insertions(+), 59 deletions(-) diff --git a/tools/ci/dynamic_pipelines/models.py b/tools/ci/dynamic_pipelines/models.py index 5565917940..299f2438f1 100644 --- a/tools/ci/dynamic_pipelines/models.py +++ b/tools/ci/dynamic_pipelines/models.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import inspect import os @@ -8,6 +8,7 @@ from dataclasses import dataclass from xml.etree.ElementTree import Element import yaml +from idf_ci_utils import IDF_PATH class Job: @@ -130,6 +131,7 @@ class TestCase: name: str file: str time: float + app_path: t.Optional[str] = None failure: t.Optional[str] = None skipped: t.Optional[str] = None ci_job_url: t.Optional[str] = None @@ -150,6 +152,13 @@ class TestCase: def is_success(self) -> bool: return not self.is_failure and not self.is_skipped + @classmethod + def _get_idf_rel_path(cls, path: str) -> str: + if path.startswith(IDF_PATH): + return os.path.relpath(path, IDF_PATH) + else: + return path + @classmethod def from_test_case_node(cls, node: Element) -> t.Optional['TestCase']: if 'name' not in node.attrib: @@ -163,6 +172,9 @@ class TestCase: kwargs = { 'name': node.attrib['name'], 'file': node.attrib.get('file'), + 'app_path': '|'.join( + cls._get_idf_rel_path(path) for path in node.attrib.get('app_path', 'unknown').split('|') + ), 'time': float(node.attrib.get('time') or 0), 'ci_job_url': node.attrib.get('ci_job_url') or 'Not found', 'ci_dashboard_url': f'{grafana_base_url}?{encoded_params}', diff --git a/tools/ci/dynamic_pipelines/report.py b/tools/ci/dynamic_pipelines/report.py index 4977544f42..232d97746a 100644 --- a/tools/ci/dynamic_pipelines/report.py +++ b/tools/ci/dynamic_pipelines/report.py @@ -788,7 +788,7 @@ class TargetTestReportGenerator(ReportGenerator): items=failed_test_cases_cur_branch, headers=[ 'Test Case', - 'Test Script File Path', + 'Test App Path', 'Failure Reason', f'Failures on your branch (40 latest testcases)', 'Dut Log URL', @@ -796,7 +796,7 @@ class TargetTestReportGenerator(ReportGenerator): 'Job URL', 'Grafana URL', ], - row_attrs=['name', 'file', 'failure', 'dut_log_url', 'ci_job_url', 'ci_dashboard_url'], + row_attrs=['name', 'app_path', 'failure', 'dut_log_url', 'ci_job_url', 'ci_dashboard_url'], value_functions=[ ( 'Failures on your branch (40 latest testcases)', @@ -813,7 +813,7 @@ class TargetTestReportGenerator(ReportGenerator): items=failed_test_cases_other_branch, headers=[ 'Test Case', - 'Test Script File Path', + 'Test App Path', 'Failure Reason', 'Cases that failed in other branches as well (40 latest testcases)', 'Dut Log URL', @@ -821,7 +821,7 @@ class TargetTestReportGenerator(ReportGenerator): 'Job URL', 'Grafana URL', ], - row_attrs=['name', 'file', 'failure', 'dut_log_url', 'ci_job_url', 'ci_dashboard_url'], + row_attrs=['name', 'app_path', 'failure', 'dut_log_url', 'ci_job_url', 'ci_dashboard_url'], value_functions=[ ( 'Cases that failed in other branches as well (40 latest testcases)', @@ -836,8 +836,8 @@ class TargetTestReportGenerator(ReportGenerator): known_failures_cases_table_section = self.create_table_section( title=self.report_titles_map['failed_known'], items=known_failures, - headers=['Test Case', 'Test Script File Path', 'Failure Reason', 'Job URL', 'Grafana URL'], - row_attrs=['name', 'file', 'failure', 'ci_job_url', 'ci_dashboard_url'], + headers=['Test Case', 'Test App Path', 'Failure Reason', 'Job URL', 'Grafana URL'], + row_attrs=['name', 'app_path', 'failure', 'ci_job_url', 'ci_dashboard_url'], ) failed_cases_report_url = self.write_report_to_file( self.generate_html_report( @@ -870,8 +870,8 @@ class TargetTestReportGenerator(ReportGenerator): skipped_cases_table_section = self.create_table_section( title=self.report_titles_map['skipped'], items=skipped_test_cases, - headers=['Test Case', 'Test Script File Path', 'Skipped Reason', 'Grafana URL'], - row_attrs=['name', 'file', 'skipped', 'ci_dashboard_url'], + headers=['Test Case', 'Test App Path', 'Skipped Reason', 'Grafana URL'], + row_attrs=['name', 'app_path', 'skipped', 'ci_dashboard_url'], ) skipped_cases_report_url = self.write_report_to_file( self.generate_html_report(''.join(skipped_cases_table_section)), @@ -892,8 +892,8 @@ class TargetTestReportGenerator(ReportGenerator): succeeded_cases_table_section = self.create_table_section( title=self.report_titles_map['succeeded'], items=succeeded_test_cases, - headers=['Test Case', 'Test Script File Path', 'Job URL', 'Grafana URL'], - row_attrs=['name', 'file', 'ci_job_url', 'ci_dashboard_url'], + headers=['Test Case', 'Test App Path', 'Job URL', 'Grafana URL'], + row_attrs=['name', 'app_path', 'ci_job_url', 'ci_dashboard_url'], ) succeeded_cases_report_url = self.write_report_to_file( self.generate_html_report(''.join(succeeded_cases_table_section)), diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index 06a63fc0bd..420ee18fec 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -56,7 +56,7 @@ TARGET_SELECTOR: "" ENV_MARKERS: "" INSTALL_EXTRA_TOOLS: "xtensa-esp-elf-gdb riscv32-esp-elf-gdb openocd-esp32 esp-rom-elfs" - PYTEST_EXTRA_FLAGS: "--dev-passwd ${ETHERNET_TEST_PASSWORD} --dev-user ${ETHERNET_TEST_USER} --capture=fd --verbosity=0" + PYTEST_EXTRA_FLAGS: "--dev-passwd ${ETHERNET_TEST_PASSWORD} --dev-user ${ETHERNET_TEST_USER} --capture=fd --verbosity=0 --unity-test-report-mode merge" cache: # Usually do not need submodule-cache in target_test - key: pip-cache-${LATEST_GIT_TAG} diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/XUNIT_REPORT.xml b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/XUNIT_REPORT.xml index 0334a68155..83a6e09767 100644 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/XUNIT_REPORT.xml +++ b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/XUNIT_REPORT.xml @@ -1,7 +1,7 @@ - + conftest.py:74: in case_tester yield CaseTester(dut, **kwargs) tools/ci/idf_unity_tester.py:202: in __init__ @@ -18,7 +18,7 @@ tools/ci/idf_unity_tester.py:202: in __init__ raise EOFError E EOFError - + conftest.py:74: in case_tester yield CaseTester(dut, **kwargs) tools/ci/idf_unity_tester.py:202: in __init__ @@ -37,9 +37,9 @@ E EOFError - - - + + + /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/plugin.py:1272: in pytest_runtest_call self._raise_dut_failed_cases_if_exists(duts) # type: ignore /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/plugin.py:1207: in _raise_dut_failed_cases_if_exists @@ -48,19 +48,19 @@ E AssertionError: Unity test failed - - + + /builds/espressif/esp-idf/tools/test_build_system/test_common.py:134: Linux does not support executing .exe files - - - - - - + + + + + + - + /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper @@ -95,7 +95,7 @@ E pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of t E Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 E Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt - + /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper @@ -128,7 +128,7 @@ E pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tes E Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) E Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt - + /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper @@ -161,7 +161,7 @@ E pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tes E Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 24528 bytes) E Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_wear_levelling/dut.txt - + /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html index d7568aa43e..1e38c58e35 100644 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html +++ b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html @@ -59,7 +59,7 @@ Test Case - Test Script File Path + Test App Path Failure Reason Cases that failed in other branches as well (40 latest testcases) Dut Log URL @@ -71,7 +71,7 @@ ('esp32h2', 'esp32h2').('defaults', 'defaults').test_i2c_multi_device - components/driver/test_apps/i2c_test_apps/pytest_i2c.py + components/driver/test_apps/i2c_test_apps failed on setup with "EOFError" 0 / 40 link @@ -81,7 +81,7 @@ esp32c3.release.test_esp_timer - components/esp_timer/test_apps/pytest_esp_timer_ut.py + components/esp_timer/test_apps pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt 0 / 40 link @@ -91,7 +91,7 @@ esp32c3.default.test_wpa_supplicant_ut - components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py + components/wpa_supplicant/test_apps pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txt 0 / 40 link @@ -101,7 +101,7 @@ ('esp32h2', 'esp32h2').('default', 'default').test_i2s_multi_dev - components/driver/test_apps/i2s_test_apps/i2s_multi_dev/pytest_i2s_multi_dev.py + components/driver/test_apps/i2s_test_apps/i2s_multi_dev failed on setup with "EOFError" 3 / 40 link @@ -111,7 +111,7 @@ esp32c2.default.test_wpa_supplicant_ut - components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py + components/wpa_supplicant/test_apps AssertionError: Unity test failed 3 / 40 link @@ -121,7 +121,7 @@ esp32c3.512safe.test_wear_levelling - components/wear_levelling/test_apps/pytest_wear_levelling.py + components/wear_levelling/test_apps pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?POK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt 3 / 40 link @@ -131,7 +131,7 @@ esp32c3.release.test_wear_levelling - components/wear_levelling/test_apps/pytest_wear_levelling.py + components/wear_levelling/test_apps pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?POK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 24528 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_wear_levelling/dut.txt 3 / 40 link @@ -145,7 +145,7 @@ Test Case - Test Script File Path + Test App Path Failure Reason Job URL Grafana URL @@ -154,28 +154,28 @@ esp32c2.default.test_wpa_supplicant_ut - components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py + components/wpa_supplicant/test_apps AssertionError: Unity test failed Not found link esp32c3.release.test_esp_timer - components/esp_timer/test_apps/pytest_esp_timer_ut.py + components/esp_timer/test_apps pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt Not found link esp32c3.512safe.test_wear_levelling - components/wear_levelling/test_apps/pytest_wear_levelling.py + components/wear_levelling/test_apps pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?POK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt Not found link esp32c3.default.test_wpa_supplicant_ut - components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py + components/wpa_supplicant/test_apps pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txt Not found link @@ -186,7 +186,7 @@ Test Case - Test Script File Path + Test App Path Skipped Reason Grafana URL @@ -194,7 +194,7 @@ test_python_interpreter_win - test_common.py + tools Linux does not support executing .exe files link @@ -204,7 +204,7 @@ Test Case - Test Script File Path + Test App Path Job URL Grafana URL @@ -212,55 +212,55 @@ esp32c2.default.test_vfs_default - components/vfs/test_apps/pytest_vfs.py + components/vfs/test_apps Not found link esp32c2.iram.test_vfs_default - components/vfs/test_apps/pytest_vfs.py + components/vfs/test_apps Not found link test_python_interpreter_unix - test_common.py + tools Not found link test_invoke_confserver - test_common.py + tools Not found link test_ccache_used_to_build - test_common.py + tools Not found link test_toolchain_prefix_in_description_file - test_common.py + tools Not found link test_subcommands_with_options - test_common.py + tools Not found link test_fallback_to_build_system_target - test_common.py + tools Not found link test_create_component_project - test_common.py + tools Not found link diff --git a/tools/ci/idf_pytest/plugin.py b/tools/ci/idf_pytest/plugin.py index 5660c8c730..06603a5a9c 100644 --- a/tools/ci/idf_pytest/plugin.py +++ b/tools/ci/idf_pytest/plugin.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import importlib import logging @@ -34,6 +34,7 @@ from .constants import SUPPORTED_TARGETS from .utils import comma_sep_str_to_list from .utils import format_case_id from .utils import merge_junit_files +from .utils import normalize_testcase_file_path IDF_PYTEST_EMBEDDED_KEY = pytest.StashKey['IdfPytestEmbedded']() ITEM_FAILED_CASES_KEY = pytest.StashKey[list]() @@ -365,16 +366,19 @@ class IdfPytestEmbedded: is_qemu = item.get_closest_marker('qemu') is not None target = item.funcargs['target'] config = item.funcargs['config'] + app_path = item.funcargs.get('app_path') for junit in junits: xml = ET.parse(junit) testcases = xml.findall('.//testcase') for case in testcases: # modify the junit files + # Use from case attrib if available, otherwise fallback to the previously defined + app_path = case.attrib.get('app_path') or app_path new_case_name = format_case_id(target, config, case.attrib['name'], is_qemu=is_qemu) case.attrib['name'] = new_case_name if 'file' in case.attrib: - case.attrib['file'] = case.attrib['file'].replace('/IDF/', '') # our unity test framework - + # our unity test framework + case.attrib['file'] = normalize_testcase_file_path(case.attrib['file'], app_path) if ci_job_url := os.getenv('CI_JOB_URL'): case.attrib['ci_job_url'] = ci_job_url diff --git a/tools/ci/idf_pytest/utils.py b/tools/ci/idf_pytest/utils.py index 1e06354668..6fa8e7502d 100644 --- a/tools/ci/idf_pytest/utils.py +++ b/tools/ci/idf_pytest/utils.py @@ -1,6 +1,5 @@ -# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 - import logging import os import typing as t @@ -65,3 +64,34 @@ def merge_junit_files(junit_files: t.List[str], target_path: str) -> None: def comma_sep_str_to_list(s: str) -> t.List[str]: return [s.strip() for s in s.split(',') if s.strip()] + + +def normalize_testcase_file_path(file: str, app_path: t.Union[str, tuple]) -> str: + """ + Normalize file paths to a consistent format, resolving relative paths based on the `app_path`. + + This function ensures that file paths are correctly resolved and normalized: + - If `app_path` is a tuple, the function will try each app path in the tuple in order and join the results with ':'. + - If `app_path` is a string, the file will be resolved using that base path. + + :param file: The original file path, which can be relative, absolute. + :param app_path: The base app path used to resolve relative paths, which can be a string or tuple. + :return: A normalized file path with the IDF_PATH prefix removed if applicable. + """ + + def normalize_path(file_path: str, app_path: str) -> str: + """Helper function to normalize a single path.""" + if not os.path.isabs(file_path): + resolved_path = os.path.normpath( + os.path.join(app_path, file_path.removeprefix('./').removeprefix('../')) + ) + else: + resolved_path = os.path.normpath(file_path) + + return resolved_path.replace(f'{os.environ.get("IDF_PATH", "")}', '').replace('/IDF/', '').lstrip('/') + + if isinstance(app_path, tuple): + normalized_paths = [normalize_path(file, base_path) for base_path in app_path] + return '|'.join(normalized_paths) + + return normalize_path(file, app_path)