From 0c69545f30e9a2facc9215bbfed458d4e076ce61 Mon Sep 17 00:00:00 2001 From: "radek.tandler" Date: Tue, 17 Dec 2024 18:35:13 +0100 Subject: [PATCH 1/4] fix(ci): Removed storage related ignore warnings --- .../nvs_flash/test_apps/pytest_nvs_flash.py | 3 ++- components/vfs/test_apps/pytest_vfs.py | 1 - .../storage/nvs_rw_blob/pytest_nvs_rw_blob.py | 2 +- .../nvs_rw_value/pytest_nvs_rw_value.py | 4 ++-- .../pytest_nvs_rw_value_cxx.py | 4 ++-- .../main/wear_levelling_example_main.c | 2 +- .../pytest_wear_levelling_example.py | 7 +++---- ..._generate_test_child_pipeline_warnings.yml | 21 +++++++++++++++++++ 8 files changed, 32 insertions(+), 12 deletions(-) create mode 100644 tools/ci/dynamic_pipelines/templates/known_generate_test_child_pipeline_warnings.yml diff --git a/components/nvs_flash/test_apps/pytest_nvs_flash.py b/components/nvs_flash/test_apps/pytest_nvs_flash.py index 174664912e..bfd5bc5941 100644 --- a/components/nvs_flash/test_apps/pytest_nvs_flash.py +++ b/components/nvs_flash/test_apps/pytest_nvs_flash.py @@ -10,7 +10,8 @@ CONFIGS_NVS_ENCR_FLASH_ENC = [ ] -@pytest.mark.supported_targets +@pytest.mark.esp32 +@pytest.mark.esp32c3 @pytest.mark.generic @pytest.mark.parametrize('config', ['default'], indirect=True) def test_nvs_flash(dut: IdfDut) -> None: diff --git a/components/vfs/test_apps/pytest_vfs.py b/components/vfs/test_apps/pytest_vfs.py index 92c19370a3..6a806483bd 100644 --- a/components/vfs/test_apps/pytest_vfs.py +++ b/components/vfs/test_apps/pytest_vfs.py @@ -18,7 +18,6 @@ def test_vfs_default(dut: Dut) -> None: @pytest.mark.esp32 -@pytest.mark.esp32s2 @pytest.mark.generic @pytest.mark.parametrize('config', [ 'ccomp', diff --git a/examples/storage/nvs_rw_blob/pytest_nvs_rw_blob.py b/examples/storage/nvs_rw_blob/pytest_nvs_rw_blob.py index 01840dcda3..78fe25a3ff 100644 --- a/examples/storage/nvs_rw_blob/pytest_nvs_rw_blob.py +++ b/examples/storage/nvs_rw_blob/pytest_nvs_rw_blob.py @@ -1,6 +1,5 @@ # SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 - import logging import random import re @@ -13,6 +12,7 @@ from pytest_embedded import Dut @pytest.mark.esp32 @pytest.mark.esp32c3 +@pytest.mark.generic def test_examples_nvs_rw_blob(dut: Dut) -> None: def expect_start_msg(index: int) -> None: dut.expect('Restart counter = {}'.format(index), timeout=10) diff --git a/examples/storage/nvs_rw_value/pytest_nvs_rw_value.py b/examples/storage/nvs_rw_value/pytest_nvs_rw_value.py index aae3f1e038..a04d33a383 100644 --- a/examples/storage/nvs_rw_value/pytest_nvs_rw_value.py +++ b/examples/storage/nvs_rw_value/pytest_nvs_rw_value.py @@ -1,6 +1,5 @@ # SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 - import logging from itertools import zip_longest @@ -10,11 +9,12 @@ from pytest_embedded import Dut @pytest.mark.esp32 @pytest.mark.esp32c3 +@pytest.mark.generic def test_examples_nvs_rw_value(dut: Dut) -> None: for i, counter_state in zip_longest(range(4), ('The value is not initialized yet!',), fillvalue='Done'): dut.expect('Opening Non-Volatile Storage \\(NVS\\) handle... Done', timeout=20) dut.expect('Reading restart counter from NVS ... {}'.format(counter_state), timeout=20) - dut.expect('Restart counter = {}'.format(i) if i > 0 else '', timeout=20) + dut.expect('Restart counter = {}'.format(i) if int(i) > 0 else '', timeout=20) dut.expect('Updating restart counter in NVS ... Done', timeout=20) dut.expect('Committing updates in NVS ... Done', timeout=20) dut.expect('Restarting in 10 seconds...', timeout=20) diff --git a/examples/storage/nvs_rw_value_cxx/pytest_nvs_rw_value_cxx.py b/examples/storage/nvs_rw_value_cxx/pytest_nvs_rw_value_cxx.py index cd1237058c..b085002ae3 100644 --- a/examples/storage/nvs_rw_value_cxx/pytest_nvs_rw_value_cxx.py +++ b/examples/storage/nvs_rw_value_cxx/pytest_nvs_rw_value_cxx.py @@ -1,6 +1,5 @@ # SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 - import logging from itertools import zip_longest @@ -10,11 +9,12 @@ from pytest_embedded import Dut @pytest.mark.esp32 @pytest.mark.esp32c3 +@pytest.mark.generic def test_examples_nvs_rw_value_cxx(dut: Dut) -> None: for i, counter_state in zip_longest(range(4), ('The value is not initialized yet!',), fillvalue='Done'): dut.expect('Opening Non-Volatile Storage \\(NVS\\) handle... Done', timeout=20) dut.expect('Reading restart counter from NVS ... {}'.format(counter_state), timeout=20) - dut.expect('Restart counter = {}'.format(i) if i > 0 else '', timeout=20) + dut.expect('Restart counter = {}'.format(i) if int(i) > 0 else '', timeout=20) dut.expect('Updating restart counter in NVS ... Done', timeout=20) dut.expect('Committing updates in NVS ... Done', timeout=20) dut.expect('Restarting in 10 seconds...', timeout=20) diff --git a/examples/storage/wear_levelling/main/wear_levelling_example_main.c b/examples/storage/wear_levelling/main/wear_levelling_example_main.c index bea0bb5d8e..db0054397f 100644 --- a/examples/storage/wear_levelling/main/wear_levelling_example_main.c +++ b/examples/storage/wear_levelling/main/wear_levelling_example_main.c @@ -105,7 +105,7 @@ void app_main(void) ESP_LOGI(TAG, "file still exists"); return; } else { - ESP_LOGI(TAG, "file doesnt exist, format done"); + ESP_LOGI(TAG, "file does not exist, format done"); } //Create file and write diff --git a/examples/storage/wear_levelling/pytest_wear_levelling_example.py b/examples/storage/wear_levelling/pytest_wear_levelling_example.py index 51ef1670ac..60b10f5e0b 100644 --- a/examples/storage/wear_levelling/pytest_wear_levelling_example.py +++ b/examples/storage/wear_levelling/pytest_wear_levelling_example.py @@ -1,7 +1,5 @@ -# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 - - import re import pytest @@ -10,6 +8,7 @@ from pytest_embedded import Dut @pytest.mark.esp32 @pytest.mark.esp32c3 +@pytest.mark.generic def test_wear_levelling_example(dut: Dut) -> None: message_list = ('example: Mounting FAT filesystem', @@ -18,7 +17,7 @@ def test_wear_levelling_example(dut: Dut) -> None: 'example: Reading file', re.compile(str.encode('example: Read from file: \'hello world, from ESP-IDF \\S+\'')), re.compile(str.encode('vfs_fat_spiflash: Formatting FATFS partition, allocation unit size=\\S+')), - 'example: file doesnt exist, format done', + 'example: file does not exist, format done', 'example: Opening file', 'example: File written', 'example: Reading file', diff --git a/tools/ci/dynamic_pipelines/templates/known_generate_test_child_pipeline_warnings.yml b/tools/ci/dynamic_pipelines/templates/known_generate_test_child_pipeline_warnings.yml new file mode 100644 index 0000000000..0167b62637 --- /dev/null +++ b/tools/ci/dynamic_pipelines/templates/known_generate_test_child_pipeline_warnings.yml @@ -0,0 +1,21 @@ +# This is the file that contains the known warnings for the generate_test_child_pipeline.py script. +# no_env_marker_test_cases: List of test cases that do not have environment markers. +# each item shall be the test node id, you may check the error message to get the node id. +no_env_marker_test_cases: + - dummy_test_case + +# no_runner_tags: List of runner tags that has no test runner set. +# each item shall be a comma separated list of runner tags. +# NOTE: +# 1. for multi dut tests, the runner tag shall be _, e.g. esp32_2 instead of esp32,esp32 +# 2. don't have spaces in the comma separated list. +no_runner_tags: + - esp32,ip101 + - esp32c2,jtag,xtal_40mhz + - esp32c3,flash_multi + - esp32c3,sdcard_sdmode + - esp32c6,jtag + - esp32h2,jtag + - esp32p4,jtag + - esp32s2,usb_host_flash_disk + From cacc13873dc46fd91e721ccdd772b23998f85e3a Mon Sep 17 00:00:00 2001 From: "radek.tandler" Date: Fri, 20 Dec 2024 19:07:08 +0100 Subject: [PATCH 2/4] fix(storage/vfs): Fixed failing test cases in test_apps --- components/nvs_flash/.build-test-rules.yml | 6 ++++++ components/vfs/.build-test-rules.yml | 5 +++++ components/vfs/test_apps/pytest_vfs.py | 3 +-- components/vfs/test_apps/sdkconfig.defaults | 2 ++ 4 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 components/vfs/.build-test-rules.yml diff --git a/components/nvs_flash/.build-test-rules.yml b/components/nvs_flash/.build-test-rules.yml index 93b76378a9..e4bf94a256 100644 --- a/components/nvs_flash/.build-test-rules.yml +++ b/components/nvs_flash/.build-test-rules.yml @@ -1,3 +1,9 @@ components/nvs_flash/host_test: enable: - if: IDF_TARGET == "linux" + +components/nvs_flash/test_apps: + disable_test: + - if: IDF_TARGET not in ["esp32", "esp32c3"] + temporary: true + reason: NVS flash test on one Xtensa and one RISCV target is enough diff --git a/components/vfs/.build-test-rules.yml b/components/vfs/.build-test-rules.yml new file mode 100644 index 0000000000..e808bae6cb --- /dev/null +++ b/components/vfs/.build-test-rules.yml @@ -0,0 +1,5 @@ +components/vfs/test_apps: + disable_test: + - if: IDF_TARGET not in ["esp32", "esp32s3", "esp32c2", "esp32c3", "esp32c6", "esp32h2"] + temporary: true + reason: Only run VFS tests on a subset of targets diff --git a/components/vfs/test_apps/pytest_vfs.py b/components/vfs/test_apps/pytest_vfs.py index 6a806483bd..f6854626f7 100644 --- a/components/vfs/test_apps/pytest_vfs.py +++ b/components/vfs/test_apps/pytest_vfs.py @@ -1,6 +1,5 @@ -# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 - import pytest from pytest_embedded import Dut diff --git a/components/vfs/test_apps/sdkconfig.defaults b/components/vfs/test_apps/sdkconfig.defaults index 728b21c79f..a600d2e6c4 100644 --- a/components/vfs/test_apps/sdkconfig.defaults +++ b/components/vfs/test_apps/sdkconfig.defaults @@ -7,3 +7,5 @@ CONFIG_PARTITION_TABLE_CUSTOM=y CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" CONFIG_ESP_TASK_WDT_INIT=n + +CONFIG_VFS_MAX_COUNT=10 From 91e0e36ffe678ce5df4bb1b6b87f4188d0256fee Mon Sep 17 00:00:00 2001 From: "radek.tandler" Date: Fri, 20 Dec 2024 19:28:11 +0100 Subject: [PATCH 3/4] fix(storage/nvs): Fixed failing test cases in example folder --- examples/storage/nvs_rw_value/pytest_nvs_rw_value.py | 2 ++ examples/storage/nvs_rw_value_cxx/pytest_nvs_rw_value_cxx.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/examples/storage/nvs_rw_value/pytest_nvs_rw_value.py b/examples/storage/nvs_rw_value/pytest_nvs_rw_value.py index a04d33a383..6d918b130a 100644 --- a/examples/storage/nvs_rw_value/pytest_nvs_rw_value.py +++ b/examples/storage/nvs_rw_value/pytest_nvs_rw_value.py @@ -11,6 +11,8 @@ from pytest_embedded import Dut @pytest.mark.esp32c3 @pytest.mark.generic def test_examples_nvs_rw_value(dut: Dut) -> None: + dut.serial.erase_flash() + dut.serial.flash() for i, counter_state in zip_longest(range(4), ('The value is not initialized yet!',), fillvalue='Done'): dut.expect('Opening Non-Volatile Storage \\(NVS\\) handle... Done', timeout=20) dut.expect('Reading restart counter from NVS ... {}'.format(counter_state), timeout=20) diff --git a/examples/storage/nvs_rw_value_cxx/pytest_nvs_rw_value_cxx.py b/examples/storage/nvs_rw_value_cxx/pytest_nvs_rw_value_cxx.py index b085002ae3..1666d62c4d 100644 --- a/examples/storage/nvs_rw_value_cxx/pytest_nvs_rw_value_cxx.py +++ b/examples/storage/nvs_rw_value_cxx/pytest_nvs_rw_value_cxx.py @@ -11,6 +11,8 @@ from pytest_embedded import Dut @pytest.mark.esp32c3 @pytest.mark.generic def test_examples_nvs_rw_value_cxx(dut: Dut) -> None: + dut.serial.erase_flash() + dut.serial.flash() for i, counter_state in zip_longest(range(4), ('The value is not initialized yet!',), fillvalue='Done'): dut.expect('Opening Non-Volatile Storage \\(NVS\\) handle... Done', timeout=20) dut.expect('Reading restart counter from NVS ... {}'.format(counter_state), timeout=20) From 791003ea7f440989a1c94238c8e564121598b52a Mon Sep 17 00:00:00 2001 From: "radek.tandler" Date: Thu, 6 Feb 2025 11:00:48 +0100 Subject: [PATCH 4/4] fix(nvs_partition_gen): Fixed sporadic failure of encryption keys --- .../nvs_partition_generator/nvs_partition_gen.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py index 36739328b7..425a521830 100755 --- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py +++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py @@ -3,24 +3,23 @@ # esp-idf NVS partition generation tool. Tool helps in generating NVS-compatible # partition binary, with key-value pair entries provided via a CSV file. # -# SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2018-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # - import argparse import array import binascii import codecs import csv import datetime -import distutils.dir_util import os import random import struct import sys import textwrap import zlib -from io import open + +import distutils.dir_util try: from cryptography.hazmat.backends import default_backend @@ -891,7 +890,7 @@ def generate_key(args): encr_key_bytes = e_key + t_key key_len = len(encr_key_bytes) - key = f"{int.from_bytes(encr_key_bytes, 'big'):x}" + key = encr_key_bytes.hex() keys_buf[0:key_len] = encr_key_bytes crc_data = keys_buf[0:key_len]