ci: target-test job skip installing toolchain, only install python env

also run with collapsed time section to better track run time

.gitlab/ci/common.yml

@@ -57,7 +57,7 @@ variables:
   # Docker images
   ESP_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-env-v5.4:1"
   ESP_IDF_DOC_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-doc-env-v5.4:1-1"
-  TARGET_TEST_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/target-test-env-v5.4:1"
+  TARGET_TEST_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/target-test-env-v5.4:2"
   SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5"
   PRE_COMMIT_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-pre-commit:1"
 
@@ -141,40 +141,48 @@ variables:
     export IDF_MIRROR_PREFIX_MAP=
   fi
 
-  # install latest python packages
+  if [[ "${CI_JOB_STAGE}" != "target_test" ]]; then
-  # target test jobs
+    section_start "running_install_sh" "Running install.sh"
-  if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then
+    if [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then
-    run_cmd bash install.sh --enable-ci --enable-pytest --enable-test-specific
+      run_cmd bash install.sh --enable-ci --enable-docs
-  elif [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then
+    elif [[ "${CI_JOB_STAGE}" == "build" ]]; then
-    run_cmd bash install.sh --enable-ci --enable-docs
-  elif [[ "${CI_JOB_STAGE}" == "build" ]]; then
-    run_cmd bash install.sh --enable-ci
-  else
-    if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then
       run_cmd bash install.sh --enable-ci
     else
-      run_cmd bash install.sh --enable-ci --enable-pytest --enable-test-specific
+      if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then
+        run_cmd bash install.sh --enable-ci
+      else
+        run_cmd bash install.sh --enable-ci --enable-pytest --enable-test-specific
+      fi
     fi
+    section_end "running_install_sh"
+  else
+    section_start "install_python_env" "Install Python environment"
+    run_cmd python tools/idf_tools.py install-python-env --features ci,pytest,test-specific
+    section_end "install_python_env"
   fi
 
-  # Install esp-clang if necessary
+  if [[ ! -z "$INSTALL_EXTRA_TOOLS" ]]; then
+    section_start "installing_optional_tools" "Install optional tools ${INSTALL_EXTRA_TOOLS}"
+    $IDF_PATH/tools/idf_tools.py --non-interactive install $INSTALL_EXTRA_TOOLS
+    section_end "installing_optional_tools"
+  fi
+
+  # Install esp-clang if necessary (esp-clang is separately installed)
   if [[ "$IDF_TOOLCHAIN" == "clang" && -z "$CI_CLANG_DISTRO_URL" ]]; then
     $IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang
   fi
 
-  # Install QEMU if necessary
+  if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then
-  if [[ ! -z "$INSTALL_QEMU" ]]; then
+    section_start "IDF_SKIP_TOOLS_CHECK" "Skip required tools check"
-    $IDF_PATH/tools/idf_tools.py --non-interactive install qemu-xtensa qemu-riscv32
+    export IDF_SKIP_TOOLS_CHECK=1
+    section_end "IDF_SKIP_TOOLS_CHECK"
   fi
-
+  section_start "source_export" "Source export.sh"
-  # Since the version 3.21 CMake passes source files and include dirs to ninja using absolute paths.
-  # Needed for pytest junit reports.
-  $IDF_PATH/tools/idf_tools.py --non-interactive install cmake
-
   source ./export.sh
+  section_end "source_export"
 
   # Custom clang toolchain
-  if [[ ! -z "$CI_CLANG_DISTRO_URL" ]]; then
+  if [[ "$IDF_TOOLCHAIN" == "clang" && ! -z "$CI_CLANG_DISTRO_URL" ]]; then
     echo "Using custom clang from ${CI_CLANG_DISTRO_URL}"
     wget $CI_CLANG_DISTRO_URL
     ARCH_NAME=$(basename $CI_CLANG_DISTRO_URL)
@@ -198,6 +206,8 @@ variables:
     rm -rf ${CI_PYTHON_TOOL_REPO}
   fi
 
+  info "setup tools and python venv done"
+
 .show_ccache_statistics: &show_ccache_statistics |
   # Show ccache statistics if enabled globally
   test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats -vv || true
@@ -222,10 +232,11 @@ variables:
     - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}"
     # remove idf-env.json, since it may contains enabled "features"
     - rm -f $IDF_TOOLS_PATH/idf-env.json
-    - $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja
     # This adds tools (compilers) and the version-specific Python environment to PATH
     - *setup_tools_and_idf_python_venv
     - fetch_submodules
+  variables:
+    INSTALL_EXTRA_TOOLS: cmake ninja
 
 .after_script:build:macos:upload-failed-job-logs:ccache-show-stats:
   after_script:

.gitlab/ci/host-test.yml

@@ -195,7 +195,7 @@ test_tools:
       junit: ${IDF_PATH}/XUNIT_*.xml
   variables:
     LC_ALL: C.UTF-8
-    INSTALL_QEMU: 1  # for test_idf_qemu.py
+    INSTALL_EXTRA_TOOLS: "qemu-xtensa qemu-riscv32"  # for test_idf_qemu.py
   script:
     - stat=0
     - cd ${IDF_PATH}/tools/ci/test_autocomplete
@@ -282,9 +282,10 @@ test_pytest_qemu:
       junit: XUNIT_RESULT.xml
   parallel:
     matrix:
-      - IDF_TARGET: [esp32, esp32c3]
+      - IDF_TARGET: "esp32"
-  variables:
+        INSTALL_EXTRA_TOOLS: "qemu-xtensa"
-    INSTALL_QEMU: 1
+      - IDF_TARGET: "esp32c3"
+        INSTALL_EXTRA_TOOLS: "qemu-riscv32"
   script:
     - run_cmd python tools/ci/ci_build_apps.py . -v
       --target $IDF_TARGET
@@ -348,7 +349,7 @@ test_pytest_macos:
   variables:
     PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1"
   script:
-    - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -vv
+    - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -v
       --target linux
       --pytest-apps
       -m \"host_test and macos_shell\"

tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml

@@ -14,6 +14,9 @@
   variables:
     # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings.
     IDF_CCACHE_ENABLE: "1"
+    # Since the version 3.21 CMake passes source files and include dirs to ninja using absolute paths.
+    # Needed for pytest junit reports.
+    INSTALL_EXTRA_TOOLS: cmake
   needs:
     - pipeline: $PARENT_PIPELINE_ID
       job: generate_build_child_pipeline
@@ -52,6 +55,7 @@
     PYTEST_NODES: ""
     TARGET_SELECTOR: ""
     ENV_MARKERS: ""
+    INSTALL_EXTRA_TOOLS: "xtensa-esp-elf-gdb riscv32-esp-elf-gdb openocd-esp32 esp-rom-elfs"
     PYTEST_EXTRA_FLAGS: "--dev-passwd ${ETHERNET_TEST_PASSWORD} --dev-user ${ETHERNET_TEST_USER} --capture=fd --verbosity=0"
   cache:
     # Usually do not need submodule-cache in target_test
@@ -71,10 +75,10 @@
     expire_in: 1 week
   script:
     # get known failure cases
-    - python tools/ci/get_known_failure_cases_file.py
+    - run_cmd python tools/ci/get_known_failure_cases_file.py
     # get runner env config file
     - retry_failed git clone $TEST_ENV_CONFIG_REPO
-    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
+    - run_cmd python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
     # CI specific options start from "--known-failure-cases-file xxx". could ignore when running locally
     - run_cmd pytest ${PYTEST_NODES}
       --target ${TARGET_SELECTOR}
@@ -86,4 +90,7 @@
       --parallel-index ${CI_NODE_INDEX:-1}
       ${PYTEST_EXTRA_FLAGS}
   after_script:
-    - python tools/ci/artifacts_handler.py upload --type logs junit_reports
+    - source tools/ci/utils.sh
+    - section_start "upload_junit_reports"
+    - run_cmd python tools/ci/artifacts_handler.py upload --type logs junit_reports
+    - section_end "upload_junit_reports"

tools/ci/utils.sh

@@ -35,7 +35,9 @@ function add_doc_server_ssh_keys() {
 }
 
 function fetch_submodules() {
+  section_start "fetch_submodules" "Fetching submodules..."
   python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}"
+  section_end "fetch_submodules"
 }
 
 function get_all_submodules() {
@@ -49,6 +51,19 @@ function set_component_ut_vars() {
   echo "exported variables COMPONENT_UT_DIRS, COMPONENT_UT_EXCLUDES"
 }
 
+# https://docs.gitlab.com/ee/ci/yaml/script.html#use-a-script-to-improve-display-of-collapsible-sections
+function section_start() {
+  local section_title="${1}"
+  local section_description="${2:-$section_title}"
+
+  echo -e "section_start:`date +%s`:${section_title}[collapsed=true]\r\e[0K${section_description}"
+}
+function section_end() {
+  local section_title="${1}"
+
+  echo -e "section_end:`date +%s`:${section_title}\r\e[0K"
+}
+
 function error() {
   printf "\033[0;31m%s\n\033[0m" "${1}" >&2
 }