blackbeard420/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf synced 2025-03-09 17:19:09 -04:00
Code Issues Projects Releases Wiki Activity

Merge branch 'ci/upload_elf_map_to_minio_v5.0' into 'release/v5.0'

Browse Source 
ci: upload elf map file to s3 server to reduce artifacts size (v5.0)

See merge request espressif/esp-idf!19902
This commit is contained in:
Fu Hanxi 2022-12-05 18:17:59 +08:00
commit 6688ba7b8e
7 changed files with 153 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

188
.gitlab/ci/build.yml
View File

@ -4,7 +4,6 @@
tags:
- build
variables:
SIZE_INFO_LOCATION: "$CI_PROJECT_DIR/size_info.txt"
# Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings.
IDF_CCACHE_ENABLE: "1"
after_script:
@ -12,36 +11,104 @@
- test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
dependencies: []
.build_pytest_template:
.build_cmake_template:
extends:
- .build_template
- .before_script_build_jobs
dependencies: # set dependencies to null to avoid missing artifacts issue
dependencies: # set dependencies to null to avoid missing artifacts issue
needs:
- job: fast_template_app
artifacts: false
tags:
- build
# build only on shiny servers since shiny storage server is at the same location
- shiny
artifacts:
paths:
- "**/build*/size.json"
- "**/build*/build_log.txt"
- "**/build*/*.bin"
# upload to s3 server to save the artifacts size
# - "**/build*/*.map"
# ttfw tests require elf files
- "**/build*/*.elf"
- "**/build*/*.map"
- "**/build*/flasher_args.json"
- "**/build*/flash_project_args"
- "**/build*/config/sdkconfig.json"
# ttfw tests require sdkconfig file
- "**/build*/sdkconfig"
- "**/build*/bootloader/*.bin"
- "**/build*/partition_table/*.bin"
- list_job_*.json
- size_info.txt
# unit test specific
- components/idf_test/unit_test/*.yml
when: always
expire_in: 4 days
after_script:
# Show ccache statistics if enabled globally
- test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
# upload the binary files to s3 server
- echo -e "\e[0Ksection_start:`date +%s`:upload_binaries_to_s3_server[collapsed=true]\r\e[0KUploading binaries to s3 Server"
- shopt -s globstar
# use || true to bypass the no-file error
- zip ${CI_JOB_ID}.zip **/build*/*.bin || true
- zip ${CI_JOB_ID}.zip **/build*/*.elf || true
- zip ${CI_JOB_ID}.zip **/build*/*.map || true
- zip ${CI_JOB_ID}.zip **/build*/flasher_args.json || true
- zip ${CI_JOB_ID}.zip **/build*/flash_project_args || true
- zip ${CI_JOB_ID}.zip **/build*/config/sdkconfig.json || true
- zip ${CI_JOB_ID}.zip **/build*/sdkconfig || true
- zip ${CI_JOB_ID}.zip **/build*/bootloader/*.bin || true
- zip ${CI_JOB_ID}.zip **/build*/partition_table/*.bin || true
- mc alias set shiny-s3 ${SHINY_S3_SERVER} ${SHINY_S3_ACCESS_KEY} ${SHINY_S3_SECRET_KEY}
- mc cp ${CI_JOB_ID}.zip shiny-s3/idf-artifacts/${CI_PIPELINE_ID}/${CI_JOB_ID}.zip
- echo -e "\e[0Ksection_end:`date +%s`:upload_binaries_to_s3_server\r\e[0K"
- echo "Please download the full binary files (including *.elf and *.map files) from the following share link"
# would be clean up after 4 days
- mc share download shiny-s3/idf-artifacts/${CI_PIPELINE_ID}/${CI_JOB_ID}.zip --expire=96h
script:
# CI specific options start from "--collect-size-info xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
-t $IDF_TARGET
--copy-sdkconfig
--collect-size-info size_info.txt
--collect-app-info list_job_${CI_NODE_INDEX:-1}.json
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--extra-preserve-dirs
examples/bluetooth/esp_ble_mesh/ble_mesh_console
examples/bluetooth/hci/controller_hci_uart_esp32
examples/wifi/iperf
.build_pytest_template:
extends:
- .build_cmake_template
- .before_script_build_jobs
artifacts:
paths:
- "**/build*/size.json"
- "**/build*/build_log.txt"
- "**/build*/*.bin"
# upload to s3 server to save the artifacts size
# - "**/build*/*.map"
# - "**/build*/*.elf"
- "**/build*/flasher_args.json"
- "**/build*/flash_project_args"
- "**/build*/config/sdkconfig.json"
- "**/build*/bootloader/*.bin"
- "**/build*/bootloader/*.elf"
- "**/build*/partition_table/*.bin"
- $SIZE_INFO_LOCATION
- list_job_*.json
- size_info.txt
when: always
expire_in: 3 days
expire_in: 4 days
script:
# CI specific options start from "--collect-size-info xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
-t $IDF_TARGET
--pytest-apps
--collect-size-info $SIZE_INFO_LOCATION
--collect-size-info size_info.txt
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
@ -49,7 +116,7 @@ build_pytest_examples_esp32:
extends:
- .build_pytest_template
- .rules:build:example_test-esp32
parallel: 3
parallel: 4
variables:
IDF_TARGET: esp32
TEST_DIR: examples
@ -67,7 +134,7 @@ build_pytest_examples_esp32s3:
extends:
- .build_pytest_template
- .rules:build:example_test-esp32s3
parallel: 3
parallel: 4
variables:
IDF_TARGET: esp32s3
TEST_DIR: examples
@ -102,6 +169,7 @@ build_pytest_components_esp32s2:
extends:
- .build_pytest_template
- .rules:build:component_ut-esp32s2
parallel: 2
variables:
IDF_TARGET: esp32s2
TEST_DIR: components
@ -110,6 +178,7 @@ build_pytest_components_esp32s3:
extends:
- .build_pytest_template
- .rules:build:component_ut-esp32s3
parallel: 2
variables:
IDF_TARGET: esp32s3
TEST_DIR: components
@ -118,6 +187,7 @@ build_pytest_components_esp32c3:
extends:
- .build_pytest_template
- .rules:build:component_ut-esp32c3
parallel: 2
variables:
IDF_TARGET: esp32c3
TEST_DIR: components
@ -139,14 +209,35 @@ build_non_test_components_apps:
# CI specific options start from "--collect-size-info xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py $COMPONENT_UT_DIRS -v
-t all
--collect-size-info $SIZE_INFO_LOCATION
--collect-size-info size_info.txt
--collect-app-info list_job_${CI_NODE_INDEX:-1}.json
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
.build_pytest_test_apps_template:
extends: .build_pytest_template
artifacts:
paths:
- "**/build*/size.json"
- "**/build*/build_log.txt"
- "**/build*/*.bin"
# upload to s3 server to save the artifacts size
# - "**/build*/*.map"
# pytest test apps requires elf files for coredump tests
- "**/build*/*.elf"
- "**/build*/flasher_args.json"
- "**/build*/flash_project_args"
- "**/build*/config/sdkconfig.json"
- "**/build*/bootloader/*.bin"
- "**/build*/partition_table/*.bin"
- list_job_*.json
- size_info.txt
when: always
expire_in: 4 days
build_pytest_test_apps_esp32:
extends:
- .build_pytest_template
- .build_pytest_test_apps_template
- .rules:build:custom_test-esp32
variables:
IDF_TARGET: esp32
@ -154,7 +245,7 @@ build_pytest_test_apps_esp32:
build_pytest_test_apps_esp32s2:
extends:
- .build_pytest_template
- .build_pytest_test_apps_template
- .rules:build:custom_test-esp32s2
variables:
IDF_TARGET: esp32s2
@ -162,7 +253,7 @@ build_pytest_test_apps_esp32s2:
build_pytest_test_apps_esp32s3:
extends:
- .build_pytest_template
- .build_pytest_test_apps_template
- .rules:build:custom_test-esp32s3
variables:
IDF_TARGET: esp32s3
@ -170,7 +261,7 @@ build_pytest_test_apps_esp32s3:
build_pytest_test_apps_esp32c3:
extends:
- .build_pytest_template
- .build_pytest_test_apps_template
- .rules:build:custom_test-esp32c3
variables:
IDF_TARGET: esp32c3
@ -178,7 +269,7 @@ build_pytest_test_apps_esp32c3:
build_pytest_test_apps_esp32c2:
extends:
- .build_pytest_template
- .build_pytest_test_apps_template
- .rules:build:custom_test-esp32c2
variables:
IDF_TARGET: esp32c2
@ -269,19 +360,17 @@ build_ssc_esp32s3:
.build_esp_idf_tests_cmake_template:
extends:
- .build_template
- .build_cmake_template
- .before_script_build_jobs
dependencies: # set dependencies to null to avoid missing artifacts issue
needs:
- job: fast_template_app
artifacts: false
artifacts:
paths:
- "**/build*/size.json"
- "**/build*/build_log.txt"
- "**/build*/*.bin"
# upload to s3 server to save the artifacts size
# - "**/build*/*.map"
# ttfw tests require elf files
- "**/build*/*.elf"
- "**/build*/*.map"
- "**/build*/flasher_args.json"
- "**/build*/flash_project_args"
- "**/build*/config/sdkconfig.json"
@ -289,12 +378,10 @@ build_ssc_esp32s3:
- "**/build*/bootloader/*.bin"
- "**/build*/partition_table/*.bin"
- list_job_*.json
- $SIZE_INFO_LOCATION
- size_info.txt
- components/idf_test/unit_test/*.yml
when: always
expire_in: 4 days
variables:
LDGEN_CHECK_MAPPING: 1
script:
# CI specific options start from "--collect-size-info xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py tools/unit-test-app -v
@ -302,7 +389,7 @@ build_ssc_esp32s3:
--config "configs/*="
--copy-sdkconfig
--preserve-all
--collect-size-info $SIZE_INFO_LOCATION
--collect-size-info size_info.txt
--collect-app-info list_job_${CI_NODE_INDEX:-1}.json
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
@ -345,52 +432,11 @@ build_esp_idf_tests_cmake_esp32c3:
variables:
IDF_TARGET: esp32c3
.build_cmake_template:
extends:
- .build_template
- .before_script_build_jobs
dependencies: # set dependencies to null to avoid missing artifacts issue
needs:
- job: fast_template_app
artifacts: false
variables:
LDGEN_CHECK_MAPPING: 1
artifacts:
paths:
- "**/build*/size.json"
- "**/build*/build_log.txt"
- "**/build*/*.bin"
- "**/build*/*.elf"
- "**/build*/*.map"
- "**/build*/flasher_args.json"
- "**/build*/flash_project_args"
- "**/build*/config/sdkconfig.json"
- "**/build*/sdkconfig"
- "**/build*/bootloader/*.bin"
- "**/build*/partition_table/*.bin"
- list_job_*.json
- $SIZE_INFO_LOCATION
when: always
expire_in: 4 days
script:
# CI specific options start from "--collect-size-info xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
-t $IDF_TARGET
--copy-sdkconfig
--collect-size-info $SIZE_INFO_LOCATION
--collect-app-info list_job_${CI_NODE_INDEX:-1}.json
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--extra-preserve-dirs
examples/bluetooth/esp_ble_mesh/ble_mesh_console
examples/bluetooth/hci/controller_hci_uart_esp32
examples/wifi/iperf
build_examples_cmake_esp32:
extends:
- .build_cmake_template
- .rules:build:example_test-esp32
parallel: 12
parallel: 9
variables:
IDF_TARGET: esp32
TEST_DIR: examples
@ -399,7 +445,7 @@ build_examples_cmake_esp32s2:
extends:
- .build_cmake_template
- .rules:build:example_test-esp32s2
parallel: 8
parallel: 6
variables:
IDF_TARGET: esp32s2
TEST_DIR: examples
@ -408,7 +454,7 @@ build_examples_cmake_esp32s3:
extends:
- .build_cmake_template
- .rules:build:example_test-esp32s3
parallel: 8
parallel: 7
variables:
IDF_TARGET: esp32s3
TEST_DIR: examples
@ -417,7 +463,7 @@ build_examples_cmake_esp32c2:
extends:
- .build_cmake_template
- .rules:build:example_test-esp32c2
parallel: 8
parallel: 4
variables:
IDF_TARGET: esp32c2
TEST_DIR: examples
@ -426,7 +472,7 @@ build_examples_cmake_esp32c3:
extends:
- .build_cmake_template
- .rules:build:example_test-esp32c3
parallel: 8
parallel: 6
variables:
IDF_TARGET: esp32c3
TEST_DIR: examples

2
.gitlab/ci/pre_check.yml
View File

@ -163,4 +163,6 @@ check_test_scripts_build_test_rules:
- .pre_check_template
- .before_script_build_jobs
script:
# required pytest related packages
- run_cmd bash install.sh --enable-pytest
- python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components

3
components/esp_pm/linker.lf
View File

@ -21,8 +21,9 @@ entries:
esp_clk:esp_rtc_get_time_us (noflash)
if GPIO_ESP32_SUPPORT_SWITCH_SLP_PULL = y:
sleep_gpio:gpio_sleep_mode_config_apply (noflash)
if IDF_TARGET_ESP32 = n && IDF_TARGET_ESP32S2 = n:
if SOC_PM_SUPPORT_CPU_PD = y || SOC_PM_SUPPORT_TAGMEM_PD = y:
sleep_retention:sleep_enable_memory_retention (noflash)
if SOC_PM_SUPPORT_CPU_PD = y:
sleep_retention:cpu_domain_pd_allowed (noflash)
[mapping:esp_system_pm]

6
components/hal/linker.lf
View File

@ -30,7 +30,8 @@ entries:
lcd_hal: lcd_hal_cal_pclk_freq (noflash)
if ADC_ONESHOT_CTRL_FUNC_IN_IRAM = y:
adc_oneshot_hal (noflash)
adc_hal_common: get_controller (noflash)
if COMPILER_OPTIMIZATION_DEFAULT = y:
adc_hal_common: get_controller (noflash)
adc_hal_common: adc_hal_set_controller (noflash)
if SOC_ADC_ARBITER_SUPPORTED = y:
adc_hal_common: adc_hal_arbiter_config (noflash)
@ -40,7 +41,8 @@ entries:
if ADC_CONTINUOUS_ISR_IRAM_SAFE = y:
adc_hal: adc_hal_get_reading_result (noflash)
adc_hal: adc_hal_digi_start (noflash)
adc_hal: adc_hal_digi_dma_link_descriptors (noflash)
if COMPILER_OPTIMIZATION_DEFAULT = y:
adc_hal: adc_hal_digi_dma_link_descriptors (noflash)
adc_hal: adc_hal_digi_stop (noflash)
if IDF_TARGET_ESP32 = y || IDF_TARGET_ESP32S2 = y:
adc_hal: adc_hal_check_event (noflash)

20
examples/get-started/blink/pytest_blink.py
View File

@ -1,7 +1,6 @@
# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: CC0-1.0
import hashlib
import logging
import os
@ -9,21 +8,6 @@ import pytest
from pytest_embedded_idf.dut import IdfDut
def verify_elf_sha256_embedding(bin_path: str, sha256_reported: str) -> None:
elf_file = os.path.join(bin_path, 'blink.elf')
sha256 = hashlib.sha256()
with open(elf_file, 'rb') as f:
sha256.update(f.read())
sha256_expected = sha256.hexdigest()
logging.info(f'ELF file SHA256: {sha256_expected}')
logging.info(f'ELF file SHA256 (reported by the app): {sha256_reported}')
# the app reports only the first several hex characters of the SHA256, check that they match
if not sha256_expected.startswith(sha256_reported):
raise ValueError('ELF file SHA256 mismatch')
@pytest.mark.supported_targets
@pytest.mark.generic
def test_blink(dut: IdfDut) -> None:
@ -31,7 +15,3 @@ def test_blink(dut: IdfDut) -> None:
binary_file = os.path.join(dut.app.binary_path, 'blink.bin')
bin_size = os.path.getsize(binary_file)
logging.info('blink_bin_size : {}KB'.format(bin_size // 1024))
sha256_reported = dut.expect(r'ELF file SHA256:\s+([a-f0-9]+)').group(1).decode('utf-8')
verify_elf_sha256_embedding(dut.app.binary_path, sha256_reported)

28
examples/get-started/hello_world/pytest_hello_world.py
View File

@ -1,22 +1,46 @@
# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: CC0-1.0
import hashlib
import logging
from typing import Callable
import pytest
from pytest_embedded_idf.dut import IdfDut
from pytest_embedded_qemu.app import QemuApp
from pytest_embedded_qemu.dut import QemuDut
@pytest.mark.supported_targets
@pytest.mark.generic
def test_hello_world(dut: IdfDut, log_minimum_free_heap_size: Callable[..., None]) -> None:
def test_hello_world(
dut: IdfDut, log_minimum_free_heap_size: Callable[..., None]
) -> None:
dut.expect('Hello world!')
log_minimum_free_heap_size()
def verify_elf_sha256_embedding(app: QemuApp, sha256_reported: str) -> None:
sha256 = hashlib.sha256()
with open(app.elf_file, 'rb') as f:
sha256.update(f.read())
sha256_expected = sha256.hexdigest()
logging.info(f'ELF file SHA256: {sha256_expected}')
logging.info(f'ELF file SHA256 (reported by the app): {sha256_reported}')
# the app reports only the first several hex characters of the SHA256, check that they match
if not sha256_expected.startswith(sha256_reported):
raise ValueError('ELF file SHA256 mismatch')
@pytest.mark.esp32 # we only support qemu on esp32 for now
@pytest.mark.host_test
@pytest.mark.qemu
def test_hello_world_host(dut: QemuDut) -> None:
def test_hello_world_host(app: QemuApp, dut: QemuDut) -> None:
sha256_reported = (
dut.expect(r'ELF file SHA256:\s+([a-f0-9]+)').group(1).decode('utf-8')
)
verify_elf_sha256_embedding(app, sha256_reported)
dut.expect('Hello world!')

2
tools/ci/configure_ci_environment.sh
View File

@ -44,3 +44,5 @@ if [ "${CI_CCACHE_DISABLE_SECONDARY}" != "1" ] && [ -n "${REDIS_CACHE}" ]; then
export CCACHE_SECONDARY_STORAGE="redis://${REDIS_CACHE}"
echo "INFO: Using CCACHE_SECONDARY_STORAGE=${CCACHE_SECONDARY_STORAGE}"
fi
export LDGEN_CHECK_MAPPING="1"