From de7e67ccec4ea8ef54d2916dd7e230660be60a29 Mon Sep 17 00:00:00 2001 From: Chen Yudong Date: Wed, 9 Feb 2022 18:00:54 +0800 Subject: [PATCH] CI: add expire_in and timeout for jobs add timeout 4 hours and retry for deploy_test_result add missing expire_in for artifacts add timeout 1 hour for tartget test jobs add retry when job_execution_timeout --- .gitlab-ci.yml | 7 +++++-- .gitlab/ci/assign-test.yml | 1 + .gitlab/ci/build.yml | 1 + .gitlab/ci/deploy.yml | 3 +++ .gitlab/ci/pre_check.yml | 1 + .gitlab/ci/static-code-analysis.yml | 1 + .gitlab/ci/target-test.yml | 5 ++++- tools/ci/check_artifacts_expire_time.py | 21 ++++++++------------- 8 files changed, 24 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 15c408c8f9..7edcf60fcb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -171,8 +171,11 @@ before_script: default: retry: max: 2 - # In case of a runner failure we could hop to another one, or a network error could go away. - when: runner_system_failure + when: + # In case of a runner failure we could hop to another one, or a network error could go away. + - runner_system_failure + # Job execution timeout may be caused by a network issue. + - job_execution_timeout include: - '.gitlab/ci/rules.yml' diff --git a/.gitlab/ci/assign-test.yml b/.gitlab/ci/assign-test.yml index d11b2a5614..0eb68b5502 100644 --- a/.gitlab/ci/assign-test.yml +++ b/.gitlab/ci/assign-test.yml @@ -88,6 +88,7 @@ assign_integration_test: artifacts: paths: - $TEST_DIR/test_configs + expire_in: 1 week variables: TEST_DIR: ${CI_PROJECT_DIR}/tools/ci/integration_test BUILD_DIR: ${CI_PROJECT_DIR}/SSC/ssc_bin diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index 86dc64b3f2..52d265375d 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -117,6 +117,7 @@ build_non_test_components_apps: - log_template_app/* - size_info.txt - build_template_app/**/size.json + expire_in: 1 week script: # Set the variable for 'esp-idf-template' testing - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"} diff --git a/.gitlab/ci/deploy.yml b/.gitlab/ci/deploy.yml index 7ab97d92da..f4e91d8fed 100644 --- a/.gitlab/ci/deploy.yml +++ b/.gitlab/ci/deploy.yml @@ -75,3 +75,6 @@ deploy_test_result: - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE} # update test results - python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l CI -p ${CI_PROJECT_DIR}/TEST_LOGS --pipeline_url ${CI_PIPELINE_URL} + # May need a long time to upload all test results. + retry: 2 + timeout: 4 hours diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index aa69df9abe..aefea4cf5b 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -152,6 +152,7 @@ scan_tests: - $EXAMPLE_TEST_OUTPUT_DIR - $TEST_APPS_OUTPUT_DIR - $COMPONENT_UT_OUTPUT_DIR + expire_in: 1 week variables: EXAMPLE_TEST_DIR: ${CI_PROJECT_DIR}/examples EXAMPLE_TEST_OUTPUT_DIR: ${CI_PROJECT_DIR}/examples/test_configs diff --git a/.gitlab/ci/static-code-analysis.yml b/.gitlab/ci/static-code-analysis.yml index 31db4ae75b..2de67d3834 100644 --- a/.gitlab/ci/static-code-analysis.yml +++ b/.gitlab/ci/static-code-analysis.yml @@ -82,6 +82,7 @@ check_pylint: when: always paths: - $REPORT_PATTERN + expire_in: 1 week tags: - host_test dependencies: # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here. diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 1fd72b5118..1f3af141e4 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -1,5 +1,6 @@ .pytest_template: stage: target_test + timeout: 1 hour extends: .before_script_pytest artifacts: when: always @@ -8,6 +9,7 @@ - /tmp/pytest-embedded/ reports: junit: XUNIT_RESULT.xml + expire_in: 1 week script: - pytest $TEST_DIR --target $TARGET -m $ENV_MARKER --junitxml=XUNIT_RESULT.xml @@ -173,6 +175,7 @@ component_ut_pytest_esp32c3_generic: .target_test_job_template: stage: target_test + timeout: 1 hour artifacts: when: always paths: @@ -180,9 +183,9 @@ component_ut_pytest_esp32c3_generic: - $LOG_PATH exclude: - .git/**/* - expire_in: 1 week reports: junit: $LOG_PATH/*/XUNIT_RESULT.xml + expire_in: 1 week variables: TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw" LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS" diff --git a/tools/ci/check_artifacts_expire_time.py b/tools/ci/check_artifacts_expire_time.py index 486d1c4abd..30ecb48e23 100644 --- a/tools/ci/check_artifacts_expire_time.py +++ b/tools/ci/check_artifacts_expire_time.py @@ -7,36 +7,31 @@ import os import yaml -try: - from yaml import CLoader as Loader -except ImportError: - from yaml import Loader as Loader - IDF_PATH = os.getenv('IDF_PATH') if not IDF_PATH: print('Please set IDF_PATH before running this script') raise SystemExit(-1) -GITLAB_CONFIG_FILE = os.path.join(os.getenv('IDF_PATH'), '.gitlab-ci.yml') +GITLAB_CONFIG_FILE = os.path.join(IDF_PATH, '.gitlab-ci.yml') def check_artifacts_expire_time(): with open(GITLAB_CONFIG_FILE, 'r') as f: - config = yaml.load(f, Loader=Loader) + config = yaml.load(f, Loader=yaml.FullLoader) - errors = [] + # load files listed in `include` + if 'include' in config: + for _file in config['include']: + with open(os.path.join(IDF_PATH, _file)) as f: + config.update(yaml.load(f, Loader=yaml.FullLoader)) print('expire time for jobs:') + errors = [] job_names = list(config.keys()) job_names.sort() for job_name in job_names: - - if job_name.startswith('.'): - # skip ignored jobs - continue - try: if 'expire_in' not in config[job_name]['artifacts']: errors.append(job_name)