From 4cbaf6fbb154c3e9a7a2c5c28c84ca3f15083b7f Mon Sep 17 00:00:00 2001
From: Fu Hanxi <fuhanxi@espressif.com>
Date: Fri, 11 Feb 2022 15:47:32 +0800
Subject: [PATCH] ci: enable pytest panic test

---
 .gitlab-ci.yml                |  2 +-
 .gitlab/ci/build.yml          | 14 ++++++++++++++
 .gitlab/ci/target-test.yml    | 36 ++++++++++++++++++++++++++++++++---
 tools/ci/build_pytest_apps.py | 30 +++++++++++++++++++++--------
 4 files changed, 70 insertions(+), 12 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5364dc6077..bb627dba95 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -73,7 +73,7 @@ variables:
   TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
   CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git"
   CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1"
-  PYTEST_EMBEDDED_TAG: "v0.5.1"
+  PYTEST_EMBEDDED_TAG: "v0.6.0rc0"
 
   # cache python dependencies
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml
index e8337e1bcd..952ad484d1 100644
--- a/.gitlab/ci/build.yml
+++ b/.gitlab/ci/build.yml
@@ -93,6 +93,20 @@ build_pytest_components_esp32c3:
   script:
     - run_cmd python tools/ci/build_pytest_apps.py components --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv
 
+build_pytest_test_apps_esp32:
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32
+  script:
+    - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32 --size-info $SIZE_INFO_LOCATION -vv
+
+build_pytest_test_apps_esp32s2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32s2
+  script:
+    - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv
+
 build_non_test_components_apps:
   extends:
     - .build_template
diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml
index 277df150b5..db1622a7db 100644
--- a/.gitlab/ci/target-test.yml
+++ b/.gitlab/ci/target-test.yml
@@ -167,6 +167,39 @@ component_ut_pytest_esp32c3_generic:
     - ESP32C3
     - COMPONENT_UT_GENERIC
 
+.pytest_test_apps_dir_template:
+  extends: .pytest_template
+  variables:
+    TEST_DIR: tools/test_apps
+
+test_app_test_pytest_esp32_generic:
+  extends:
+    - .pytest_test_apps_dir_template
+    - .rules:test:custom_test-esp32
+  needs:
+    - build_pytest_test_apps_esp32
+  variables:
+    TARGET: esp32
+    ENV_MARKER: generic
+    SETUP_TOOLS: "1"  # need gdb
+  tags:
+    - ESP32
+    - Example_GENERIC
+
+test_app_test_pytest_esp32s2_generic:
+  extends:
+    - .pytest_test_apps_dir_template
+    - .rules:test:custom_test-esp32s2
+  needs:
+    - build_pytest_test_apps_esp32s2
+  variables:
+    TARGET: esp32s2
+    ENV_MARKER: generic
+    SETUP_TOOLS: "1"  # need gdb
+  tags:
+    - ESP32S2
+    - Example_GENERIC
+
 # for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2")
 # we need to convert to pattern "job_name_index.yml"
 .define_config_file_name: &define_config_file_name |
@@ -538,12 +571,9 @@ test_app_test_005:
 
 test_app_test_esp32_generic:
   extends: .test_app_esp32_template
-  parallel: 5
   tags:
     - ESP32
     - Example_GENERIC
-  variables:
-    SETUP_TOOLS: "1"
 
 test_app_test_flash_psram_f4r4:
   extends: .test_app_esp32s3_template
diff --git a/tools/ci/build_pytest_apps.py b/tools/ci/build_pytest_apps.py
index e5683d17fa..8de9964339 100644
--- a/tools/ci/build_pytest_apps.py
+++ b/tools/ci/build_pytest_apps.py
@@ -6,6 +6,7 @@ This file is used to generate binary files for the given path.
 """
 
 import argparse
+import copy
 import logging
 import os
 import sys
@@ -58,17 +59,25 @@ def main(args: argparse.Namespace) -> None:
             build_system='cmake',
             config_rules=config_rules,
         )
-    logging.info(f'Found {len(build_items)} builds')
-    build_items.sort(key=lambda x: x.build_path)  # type: ignore
 
+    modified_build_items = []
     # auto clean up the binaries if no flag --preserve-all
-    if args.preserve_all is False:
-        for item in build_items:
-            if item.config_name not in app_configs[item.app_dir]:
-                item.preserve = False
+    for item in build_items:
+        is_test_related = item.config_name in app_configs[item.app_dir]
+        if args.test_only and not is_test_related:
+            logging.info(f'Skipping non-test app: {item}')
+            continue
+
+        copied_item = copy.deepcopy(item)
+        if not args.preserve_all and not is_test_related:
+            copied_item.preserve = False
+        modified_build_items.append(copied_item)
+
+    logging.info(f'Found {len(modified_build_items)} builds')
+    modified_build_items.sort(key=lambda x: x.build_path)  # type: ignore
 
     build_apps(
-        build_items=build_items,
+        build_items=modified_build_items,
         parallel_count=args.parallel_count,
         parallel_index=args.parallel_index,
         dry_run=False,
@@ -128,7 +137,12 @@ if __name__ == '__main__':
     parser.add_argument(
         '--preserve-all',
         action='store_true',
-        help='add this flag to preserve the binaries for all apps',
+        help='Preserve the binaries for all apps when specified.',
+    )
+    parser.add_argument(
+        '--test-only',
+        action='store_true',
+        help='Build only test related app when specified.',
     )
     arguments = parser.parse_args()
     setup_logging(arguments)