CI: add expire_in and timeout for jobs

add timeout 4 hours and retry for deploy_test_result
add missing expire_in for artifacts
add timeout 1 hour for tartget test jobs
add retry when job_execution_timeout

.gitlab-ci.yml

@@ -171,8 +171,11 @@ before_script:
 default:
   retry:
     max: 2
-    # In case of a runner failure we could hop to another one, or a network error could go away.
+    when:
-    when: runner_system_failure
+      # In case of a runner failure we could hop to another one, or a network error could go away.
+      - runner_system_failure
+      # Job execution timeout may be caused by a network issue.
+      - job_execution_timeout
 
 include:
   - '.gitlab/ci/rules.yml'

.gitlab/ci/assign-test.yml

@@ -88,6 +88,7 @@ assign_integration_test:
   artifacts:
     paths:
       - $TEST_DIR/test_configs
+    expire_in: 1 week
   variables:
     TEST_DIR: ${CI_PROJECT_DIR}/tools/ci/integration_test
     BUILD_DIR: ${CI_PROJECT_DIR}/SSC/ssc_bin

.gitlab/ci/build.yml

@@ -117,6 +117,7 @@ build_non_test_components_apps:
       - log_template_app/*
       - size_info.txt
       - build_template_app/**/size.json
+    expire_in: 1 week
   script:
     # Set the variable for 'esp-idf-template' testing
     - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}

.gitlab/ci/deploy.yml

@@ -75,3 +75,6 @@ deploy_test_result:
     - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
     # update test results
     - python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l CI -p ${CI_PROJECT_DIR}/TEST_LOGS --pipeline_url ${CI_PIPELINE_URL}
+  # May need a long time to upload all test results.
+  retry: 2
+  timeout: 4 hours

.gitlab/ci/pre_check.yml

@@ -152,6 +152,7 @@ scan_tests:
       - $EXAMPLE_TEST_OUTPUT_DIR
       - $TEST_APPS_OUTPUT_DIR
       - $COMPONENT_UT_OUTPUT_DIR
+    expire_in: 1 week
   variables:
     EXAMPLE_TEST_DIR: ${CI_PROJECT_DIR}/examples
     EXAMPLE_TEST_OUTPUT_DIR: ${CI_PROJECT_DIR}/examples/test_configs

.gitlab/ci/static-code-analysis.yml

@@ -82,6 +82,7 @@ check_pylint:
     when: always
     paths:
       - $REPORT_PATTERN
+    expire_in: 1 week
   tags:
     - host_test
   dependencies:  # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here.

.gitlab/ci/target-test.yml

@@ -1,5 +1,6 @@
 .pytest_template:
   stage: target_test
+  timeout: 1 hour
   extends: .before_script_pytest
   artifacts:
     when: always
@@ -8,6 +9,7 @@
       - /tmp/pytest-embedded/
     reports:
       junit: XUNIT_RESULT.xml
+    expire_in: 1 week
   script:
     - pytest $TEST_DIR --target $TARGET -m $ENV_MARKER --junitxml=XUNIT_RESULT.xml
 
@@ -173,6 +175,7 @@ component_ut_pytest_esp32c3_generic:
 
 .target_test_job_template:
   stage: target_test
+  timeout: 1 hour
   artifacts:
     when: always
     paths:
@@ -180,9 +183,9 @@ component_ut_pytest_esp32c3_generic:
       - $LOG_PATH
     exclude:
       - .git/**/*
-    expire_in: 1 week
     reports:
       junit: $LOG_PATH/*/XUNIT_RESULT.xml
+    expire_in: 1 week
   variables:
     TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
     LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"

tools/ci/check_artifacts_expire_time.py

@@ -7,36 +7,31 @@ import os
 
 import yaml
 
-try:
-    from yaml import CLoader as Loader
-except ImportError:
-    from yaml import Loader as Loader
-
 IDF_PATH = os.getenv('IDF_PATH')
 if not IDF_PATH:
     print('Please set IDF_PATH before running this script')
     raise SystemExit(-1)
 
-GITLAB_CONFIG_FILE = os.path.join(os.getenv('IDF_PATH'), '.gitlab-ci.yml')
+GITLAB_CONFIG_FILE = os.path.join(IDF_PATH, '.gitlab-ci.yml')
 
 
 def check_artifacts_expire_time():
     with open(GITLAB_CONFIG_FILE, 'r') as f:
-        config = yaml.load(f, Loader=Loader)
+        config = yaml.load(f, Loader=yaml.FullLoader)
 
-    errors = []
+    # load files listed in `include`
+    if 'include' in config:
+        for _file in config['include']:
+            with open(os.path.join(IDF_PATH, _file)) as f:
+                config.update(yaml.load(f, Loader=yaml.FullLoader))
 
     print('expire time for jobs:')
+    errors = []
 
     job_names = list(config.keys())
     job_names.sort()
 
     for job_name in job_names:
-
-        if job_name.startswith('.'):
-            # skip ignored jobs
-            continue
-
         try:
             if 'expire_in' not in config[job_name]['artifacts']:
                 errors.append(job_name)