mirror of
https://github.com/espressif/esp-idf
synced 2025-03-09 09:09:10 -04:00
ci(rules): auto generate rules.yml labels titles and rules
Also simplified the labels and ci - Remove label regular_test, weekend_test - Remove apply_job_filter - check_submodule_sync: only run on protected branch
This commit is contained in:
Fu Hanxi
parent
8ff6461b4c
commit
b24b06d16c
@ -5,7 +5,6 @@ stages:
|
||||
- host_test
|
||||
- target_test
|
||||
- test_deploy
|
||||
- post_check
|
||||
- deploy
|
||||
- post_deploy
|
||||
|
||||
@ -50,7 +49,7 @@ variables:
|
||||
BOT_DOCKER_IMAGE_TAG: ":latest"
|
||||
|
||||
# target test config file, used by assign test job
|
||||
CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/tools/ci/config/target-test.yml"
|
||||
CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/.gitlab/ci/target-test.yml"
|
||||
|
||||
# target test repo parameters
|
||||
TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
|
||||
@ -71,38 +70,27 @@ variables:
|
||||
before_script:
|
||||
- source tools/ci/utils.sh
|
||||
- source tools/ci/setup_python.sh
|
||||
- apply_bot_filter
|
||||
- add_gitlab_ssh_keys
|
||||
- source tools/ci/configure_ci_environment.sh
|
||||
- *setup_tools_unless_target_test
|
||||
- fetch_submodules
|
||||
|
||||
# used for check scripts which we want to run unconditionally
|
||||
.before_script_lesser_nofilter:
|
||||
before_script:
|
||||
- echo "Not setting up GitLab key, not fetching submodules, not applying bot filter"
|
||||
- source tools/ci/utils.sh
|
||||
- source tools/ci/setup_python.sh
|
||||
- source tools/ci/configure_ci_environment.sh
|
||||
|
||||
# used for everything else where we want to do no prep, except for bot filter
|
||||
.before_script_lesser:
|
||||
.before_script_no_sync_submodule:
|
||||
before_script:
|
||||
- echo "Not setting up GitLab key, not fetching submodules"
|
||||
- source tools/ci/utils.sh
|
||||
- source tools/ci/setup_python.sh
|
||||
- apply_bot_filter
|
||||
- source tools/ci/configure_ci_environment.sh
|
||||
|
||||
.before_script_slim:
|
||||
.before_script_minimal:
|
||||
before_script:
|
||||
- echo "Only load utils.sh inside"
|
||||
- echo "Only load utils.sh"
|
||||
- source tools/ci/utils.sh
|
||||
|
||||
.before_script_macos:
|
||||
before_script:
|
||||
- source tools/ci/utils.sh
|
||||
- apply_bot_filter
|
||||
- $IDF_PATH/tools/idf_tools.py install-python-env
|
||||
# On macOS, these tools need to be installed
|
||||
- $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja
|
||||
@ -116,13 +104,12 @@ before_script:
|
||||
- fetch_submodules
|
||||
|
||||
include:
|
||||
- '/tools/ci/config/rules.yml'
|
||||
- '/tools/ci/config/docs.yml'
|
||||
- '/tools/ci/config/static-code-analysis.yml'
|
||||
- '/tools/ci/config/pre_check.yml'
|
||||
- '/tools/ci/config/build.yml'
|
||||
- '/tools/ci/config/assign-test.yml'
|
||||
- '/tools/ci/config/host-test.yml'
|
||||
- '/tools/ci/config/target-test.yml'
|
||||
- '/tools/ci/config/post_check.yml'
|
||||
- '/tools/ci/config/deploy.yml'
|
||||
- '.gitlab/ci/rules.yml'
|
||||
- '.gitlab/ci/docs.yml'
|
||||
- '.gitlab/ci/static-code-analysis.yml'
|
||||
- '.gitlab/ci/pre_check.yml'
|
||||
- '.gitlab/ci/build.yml'
|
||||
- '.gitlab/ci/assign-test.yml'
|
||||
- '.gitlab/ci/host-test.yml'
|
||||
- '.gitlab/ci/target-test.yml'
|
||||
- '.gitlab/ci/deploy.yml'
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
assign_test:
|
||||
extends:
|
||||
- .rules:assign_test:target_test-integration_test-weekend_test
|
||||
extends: .rules:test:any_test
|
||||
tags:
|
||||
- assign_test
|
||||
image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
|
||||
|
||||
@ -8,9 +8,7 @@
|
||||
dependencies: []
|
||||
|
||||
.build_template_app_template:
|
||||
extends:
|
||||
- .build_template
|
||||
- .rules:labels:build
|
||||
extends: .build_template
|
||||
variables:
|
||||
LOG_PATH: "${CI_PROJECT_DIR}/log_template_app"
|
||||
BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app"
|
||||
@ -46,7 +44,7 @@
|
||||
fast_template_app:
|
||||
extends:
|
||||
- .build_template_app_template
|
||||
- .rules:build_tests:target_test-weekend_test
|
||||
- .rules:test:target_test
|
||||
stage: pre_check
|
||||
variables:
|
||||
BUILD_COMMAND_ARGS: "-p"
|
||||
@ -55,7 +53,7 @@ fast_template_app:
|
||||
.build_ssc_template:
|
||||
extends:
|
||||
- .build_template
|
||||
- .rules:build_tests:integration_test
|
||||
- .rules:build:integration_test
|
||||
artifacts:
|
||||
paths:
|
||||
- SSC/ssc_bin
|
||||
@ -84,10 +82,8 @@ build_ssc_esp32c3:
|
||||
variables:
|
||||
TARGET_NAME: "ESP32C3"
|
||||
|
||||
.build_esp_idf_tests_cmake:
|
||||
extends:
|
||||
- .build_template
|
||||
- .rules:build_tests:unit_test
|
||||
.build_esp_idf_tests_cmake_template:
|
||||
extends: .build_template
|
||||
dependencies: # set dependencies to null to avoid missing artifacts issue
|
||||
needs:
|
||||
- job: fast_template_app
|
||||
@ -117,29 +113,35 @@ build_ssc_esp32c3:
|
||||
- python tools/UnitTestParser.py ${BUILD_PATH}
|
||||
|
||||
build_esp_idf_tests_cmake_esp32:
|
||||
extends: .build_esp_idf_tests_cmake
|
||||
extends:
|
||||
- .build_esp_idf_tests_cmake_template
|
||||
- .rules:build:unit_test-esp32
|
||||
variables:
|
||||
IDF_TARGET: esp32
|
||||
|
||||
build_esp_idf_tests_cmake_esp32s2:
|
||||
extends: .build_esp_idf_tests_cmake
|
||||
extends:
|
||||
- .build_esp_idf_tests_cmake_template
|
||||
- .rules:build:unit_test-esp32s2
|
||||
variables:
|
||||
IDF_TARGET: esp32s2
|
||||
|
||||
build_esp_idf_tests_cmake_esp32s3:
|
||||
extends: .build_esp_idf_tests_cmake
|
||||
extends:
|
||||
- .build_esp_idf_tests_cmake_template
|
||||
- .rules:build:unit_test-esp32s3
|
||||
variables:
|
||||
IDF_TARGET: esp32s3
|
||||
|
||||
build_esp_idf_tests_cmake_esp32c3:
|
||||
extends: .build_esp_idf_tests_cmake
|
||||
extends:
|
||||
- .build_esp_idf_tests_cmake_template
|
||||
- .rules:build:unit_test-esp32c3
|
||||
variables:
|
||||
IDF_TARGET: esp32c3
|
||||
|
||||
.build_examples_template:
|
||||
extends:
|
||||
- .build_template
|
||||
- .rules:build_tests:example_test-weekend_test
|
||||
extends: .build_template
|
||||
dependencies: # set dependencies to null to avoid missing artifacts issue
|
||||
needs:
|
||||
- job: fast_template_app
|
||||
@ -162,6 +164,7 @@ build_esp_idf_tests_cmake_esp32c3:
|
||||
build_examples_make:
|
||||
extends:
|
||||
- .build_examples_template
|
||||
- .rules:build:example_test-esp32
|
||||
# This is a workaround for a rarely encountered issue with building examples in CI.
|
||||
# Probably related to building of Kconfig in 'make clean' stage
|
||||
retry: 1
|
||||
@ -178,7 +181,7 @@ build_examples_make:
|
||||
IDF_TARGET: esp32 # currently we only support esp32
|
||||
|
||||
# same as above, but for CMake
|
||||
.build_examples_cmake:
|
||||
.build_examples_cmake_template:
|
||||
extends: .build_examples_template
|
||||
artifacts:
|
||||
paths:
|
||||
@ -200,27 +203,31 @@ build_examples_make:
|
||||
BUILD_SYSTEM: cmake
|
||||
|
||||
build_examples_cmake_esp32:
|
||||
extends: .build_examples_cmake
|
||||
extends:
|
||||
- .build_examples_cmake_template
|
||||
- .rules:build:example_test-esp32
|
||||
parallel: 10
|
||||
variables:
|
||||
IDF_TARGET: esp32
|
||||
|
||||
build_examples_cmake_esp32s2:
|
||||
extends: .build_examples_cmake
|
||||
extends:
|
||||
- .build_examples_cmake_template
|
||||
- .rules:build:example_test-esp32s2
|
||||
parallel: 8
|
||||
variables:
|
||||
IDF_TARGET: esp32s2
|
||||
|
||||
build_examples_cmake_esp32c3:
|
||||
extends: .build_examples_cmake
|
||||
extends:
|
||||
- .build_examples_cmake_template
|
||||
- .rules:build:example_test-esp32c3
|
||||
parallel: 8
|
||||
variables:
|
||||
IDF_TARGET: esp32c3
|
||||
|
||||
.build_test_apps:
|
||||
extends:
|
||||
- .build_examples_cmake
|
||||
- .rules:build_tests:custom_test-weekend_test
|
||||
.build_test_apps_template:
|
||||
extends: .build_examples_cmake_template
|
||||
variables:
|
||||
TEST_PREFIX: test_apps
|
||||
TEST_RELATIVE_DIR: tools/test_apps
|
||||
@ -229,57 +236,68 @@ build_examples_cmake_esp32c3:
|
||||
- ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
|
||||
|
||||
build_test_apps_esp32:
|
||||
extends: .build_test_apps
|
||||
extends:
|
||||
- .build_test_apps_template
|
||||
- .rules:build:custom_test-esp32
|
||||
parallel: 8
|
||||
variables:
|
||||
IDF_TARGET: esp32
|
||||
|
||||
build_test_apps_esp32s2:
|
||||
extends: .build_test_apps
|
||||
extends:
|
||||
- .build_test_apps_template
|
||||
- .rules:build:custom_test-esp32s2
|
||||
parallel: 8
|
||||
variables:
|
||||
IDF_TARGET: esp32s2
|
||||
|
||||
build_test_apps_esp32s3:
|
||||
extends: .build_test_apps
|
||||
extends:
|
||||
- .build_test_apps_template
|
||||
- .rules:build:custom_test-esp32s3
|
||||
parallel: 8
|
||||
variables:
|
||||
IDF_TARGET: esp32s3
|
||||
|
||||
build_test_apps_esp32c3:
|
||||
extends: .build_test_apps
|
||||
extends:
|
||||
- .build_test_apps_template
|
||||
- .rules:build:custom_test-esp32c3
|
||||
parallel: 8
|
||||
variables:
|
||||
IDF_TARGET: esp32c3
|
||||
|
||||
.build_component_ut:
|
||||
extends:
|
||||
- .build_test_apps
|
||||
- .rules:build_tests:unit_test
|
||||
.build_component_ut_template:
|
||||
extends: .build_test_apps_template
|
||||
variables:
|
||||
TEST_PREFIX: component_ut
|
||||
TEST_RELATIVE_DIR: component_ut
|
||||
|
||||
build_component_ut_esp32:
|
||||
extends: .build_component_ut
|
||||
extends:
|
||||
- .build_component_ut_template
|
||||
- .rules:build:component_ut-esp32
|
||||
variables:
|
||||
IDF_TARGET: esp32
|
||||
|
||||
build_component_ut_esp32s2:
|
||||
extends: .build_component_ut
|
||||
extends:
|
||||
- .build_component_ut_template
|
||||
- .rules:build:component_ut-esp32s2
|
||||
variables:
|
||||
IDF_TARGET: esp32s2
|
||||
|
||||
build_component_ut_esp32c3:
|
||||
extends: .build_component_ut
|
||||
extends:
|
||||
- .build_component_ut_template
|
||||
- .rules:build:component_ut-esp32c3
|
||||
variables:
|
||||
IDF_TARGET: esp32c3
|
||||
|
||||
.test_build_system_template:
|
||||
extends:
|
||||
- .build_template
|
||||
- .rules:build_tests:weekend_test
|
||||
dependencies: # set dependencies to null to avoid missing artifacts issue
|
||||
- .rules:build
|
||||
needs:
|
||||
- job: fast_template_app
|
||||
artifacts: false
|
||||
@ -304,7 +322,7 @@ test_build_system_cmake_macos:
|
||||
extends:
|
||||
- .test_build_system_template
|
||||
- .before_script_macos
|
||||
- .rules:os:mac_os
|
||||
- .rules:build:macos
|
||||
tags:
|
||||
- macos_shell
|
||||
variables:
|
||||
@ -312,8 +330,8 @@ test_build_system_cmake_macos:
|
||||
|
||||
build_docker:
|
||||
extends:
|
||||
- .before_script_slim
|
||||
- .rules:protected-schedule
|
||||
- .before_script_minimal
|
||||
- .rules:build:docker
|
||||
stage: build
|
||||
image: espressif/docker-builder:1
|
||||
tags:
|
||||
@ -334,8 +352,8 @@ build_docker:
|
||||
|
||||
.test-on-windows:
|
||||
extends:
|
||||
- .before_script_slim
|
||||
- .rules:protected-schedule
|
||||
- .before_script_minimal
|
||||
- .rules:build:windows
|
||||
stage: build
|
||||
image: $CI_DOCKER_REGISTRY/esp32-toolchain-win-cross
|
||||
tags:
|
||||
@ -366,14 +384,10 @@ build_cmdlinerunner:
|
||||
TEST_DIR: tools/windows/tool_setup/cmdlinerunner
|
||||
|
||||
build_installer:
|
||||
extends:
|
||||
- .before_script_slim
|
||||
- .rules:protected-schedule
|
||||
extends: .test-on-windows
|
||||
# using a different stage here to be able to use artifacts from build_cmdlinerunner job
|
||||
stage: host_test
|
||||
image: $CI_DOCKER_REGISTRY/wine-innosetup:1
|
||||
tags:
|
||||
- build
|
||||
dependencies: # set dependencies to null to avoid missing artifacts issue
|
||||
needs:
|
||||
- build_cmdlinerunner
|
||||
@ -383,7 +397,9 @@ build_installer:
|
||||
|
||||
# This job builds template app with permutations of targets and optimization levels
|
||||
build_template_app:
|
||||
extends:
|
||||
- .build_template_app_template
|
||||
- .rules:build
|
||||
needs:
|
||||
- job: fast_template_app
|
||||
artifacts: false
|
||||
extends: .build_template_app_template
|
||||
|
||||
142
.gitlab/ci/dependencies/dependencies.yml
Normal file
142
.gitlab/ci/dependencies/dependencies.yml
Normal file
@ -0,0 +1,142 @@
|
||||
.all_targets: &all_targets
|
||||
- esp32
|
||||
- esp32s2
|
||||
- esp32s3
|
||||
- esp32c3
|
||||
|
||||
.target_test: &target_test
|
||||
- example_test
|
||||
- custom_test
|
||||
- unit_test
|
||||
- component_ut
|
||||
|
||||
"build:{0}-{1}":
|
||||
matrix:
|
||||
- *target_test
|
||||
- *all_targets
|
||||
labels:
|
||||
- build
|
||||
patterns:
|
||||
- build
|
||||
|
||||
"build:example_test-esp32":
|
||||
labels:
|
||||
- build
|
||||
- weekend_test # only have esp32 jobs
|
||||
- iperf_stress_test # only have esp32 jobs
|
||||
patterns:
|
||||
- build
|
||||
- example_test
|
||||
|
||||
"build:{0}":
|
||||
matrix:
|
||||
- [windows, docker]
|
||||
labels:
|
||||
- build
|
||||
- "{0}"
|
||||
patterns:
|
||||
- build
|
||||
- "{0}"
|
||||
|
||||
"build:macos":
|
||||
labels:
|
||||
- build
|
||||
- macos
|
||||
- macos_test # for backward compatibility
|
||||
patterns:
|
||||
- build
|
||||
- macos
|
||||
|
||||
"build:docs":
|
||||
labels:
|
||||
- build
|
||||
- docs
|
||||
- build_docs # for backward compatibility
|
||||
patterns:
|
||||
- docs
|
||||
deploy:
|
||||
- preview
|
||||
- production
|
||||
|
||||
"build":
|
||||
labels:
|
||||
- build
|
||||
patterns:
|
||||
- build
|
||||
|
||||
"test:{0}-{1}":
|
||||
matrix:
|
||||
- *target_test
|
||||
- *all_targets
|
||||
labels:
|
||||
- "{0}"
|
||||
- "{0}_{1}"
|
||||
patterns:
|
||||
- "{0}"
|
||||
- build
|
||||
- "build-{0}"
|
||||
included_in:
|
||||
- "build:{0}-{1}"
|
||||
- test:target_test
|
||||
- test:any_test
|
||||
|
||||
"test:component_ut-{0}":
|
||||
matrix:
|
||||
- *all_targets
|
||||
labels:
|
||||
- component_ut
|
||||
- "component_ut_{0}"
|
||||
- unit_test
|
||||
- "unit_test_{0}"
|
||||
patterns:
|
||||
- component_ut
|
||||
- build
|
||||
- "build-component_ut-{0}"
|
||||
included_in:
|
||||
- "build:component_ut-{0}"
|
||||
- test:target_test
|
||||
- test:any_test
|
||||
|
||||
# due to the lack of runners, c3 tests will only be triggered by label
|
||||
"test:unit_test-esp32c3":
|
||||
labels:
|
||||
- unit_test_esp32c3
|
||||
patterns:
|
||||
- unit_test
|
||||
- build
|
||||
- "build-unit_test-esp32c3"
|
||||
included_in:
|
||||
- "build:unit_test-esp32c3"
|
||||
|
||||
"test:integration_test":
|
||||
labels:
|
||||
- "integration_test"
|
||||
patterns:
|
||||
- "integration_test"
|
||||
included_in:
|
||||
- "build:integration_test"
|
||||
- test:target_test
|
||||
- test:any_test
|
||||
|
||||
"test:host_test":
|
||||
labels:
|
||||
- host_test
|
||||
patterns:
|
||||
- host_test
|
||||
included_in:
|
||||
- test:any_test
|
||||
|
||||
"labels:{0}":
|
||||
matrix:
|
||||
- [weekend_test, iperf_stress_test, nvs_coverage]
|
||||
labels:
|
||||
- "{0}"
|
||||
included_in:
|
||||
- test:any_test
|
||||
|
||||
"labels:fuzzer_test-weekend_test":
|
||||
labels:
|
||||
- fuzzer_test
|
||||
- weekend_test
|
||||
included_in:
|
||||
- test:any_test
|
||||
303
.gitlab/ci/dependencies/generate_rules.py
Executable file
303
.gitlab/ci/dependencies/generate_rules.py
Executable file
@ -0,0 +1,303 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright 2021 Espressif Systems (Shanghai) CO LTD
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
import inspect
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from itertools import product
|
||||
|
||||
try:
|
||||
import pygraphviz as pgv
|
||||
except ImportError: # used when pre-commit, skip generating image
|
||||
pass
|
||||
|
||||
import yaml
|
||||
|
||||
IDF_PATH = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..', '..')))
|
||||
|
||||
|
||||
def _list(str_or_list):
|
||||
if isinstance(str_or_list, str):
|
||||
return [str_or_list]
|
||||
elif isinstance(str_or_list, list):
|
||||
return str_or_list
|
||||
else:
|
||||
raise ValueError('Wrong type: {}. Only supports str or list.'.format(type(str_or_list)))
|
||||
|
||||
|
||||
def _format_nested_dict(_dict, f_tuple):
|
||||
res = {}
|
||||
for k, v in _dict.items():
|
||||
k = k.split('__')[0]
|
||||
if isinstance(v, dict):
|
||||
v = _format_nested_dict(v, f_tuple)
|
||||
elif isinstance(v, list):
|
||||
v = _format_nested_list(v, f_tuple)
|
||||
elif isinstance(v, str):
|
||||
v = v.format(*f_tuple)
|
||||
res[k.format(*f_tuple)] = v
|
||||
return res
|
||||
|
||||
|
||||
def _format_nested_list(_list, f_tuple):
|
||||
res = []
|
||||
for item in _list:
|
||||
if isinstance(item, list):
|
||||
item = _format_nested_list(item, f_tuple)
|
||||
elif isinstance(item, dict):
|
||||
item = _format_nested_dict(item, f_tuple)
|
||||
elif isinstance(item, str):
|
||||
item = item.format(*f_tuple)
|
||||
res.append(item)
|
||||
return res
|
||||
|
||||
|
||||
class RulesWriter:
|
||||
AUTO_GENERATE_MARKER = inspect.cleandoc(r'''
|
||||
##################
|
||||
# Auto Generated #
|
||||
##################
|
||||
''')
|
||||
|
||||
LABEL_TEMPLATE = inspect.cleandoc(r'''
|
||||
.if-label-{0}: &if-label-{0}
|
||||
if: '$BOT_LABEL_{1}'
|
||||
''')
|
||||
TITLE_TEMPLATE = inspect.cleandoc(r'''
|
||||
.if-title-{0}: &if-title-{0}
|
||||
if: '$CI_MERGE_REQUEST_LABELS =~ /^(?:\w+,)*{0}(?:,\w+)*$/i || $CI_COMMIT_DESCRIPTION =~ /test labels?: (?:\w+[, ]+)*{0}(?:[, ]+\w+)*/i'
|
||||
''')
|
||||
|
||||
RULE_NORM = ' - <<: *if-protected'
|
||||
RULE_PROD = ' - <<: *if-protected-no_label'
|
||||
RULE_LABEL_TEMPLATE = ' - <<: *if-label-{0}'
|
||||
RULE_TITLE_TEMPLATE = ' - <<: *if-title-{0}'
|
||||
RULE_PATTERN_TEMPLATE = ' - <<: *if-dev-push\n' \
|
||||
' changes: *patterns-{0}'
|
||||
RULES_TEMPLATE = inspect.cleandoc(r"""
|
||||
.rules:{0}:
|
||||
rules:
|
||||
{1}
|
||||
""")
|
||||
|
||||
KEYWORDS = ['labels', 'patterns']
|
||||
|
||||
def __init__(self, rules_yml, depend_yml): # type: (str, str) -> None
|
||||
self.rules_yml = rules_yml
|
||||
self.rules_cfg = yaml.load(open(rules_yml), Loader=yaml.FullLoader)
|
||||
|
||||
self.full_cfg = yaml.load(open(depend_yml), Loader=yaml.FullLoader)
|
||||
self.cfg = {k: v for k, v in self.full_cfg.items() if not k.startswith('.')}
|
||||
self.cfg = self.expand_matrices()
|
||||
self.rules = self.expand_rules()
|
||||
|
||||
self.graph = None
|
||||
|
||||
def expand_matrices(self): # type: () -> dict
|
||||
"""
|
||||
Expand the matrix into different rules
|
||||
"""
|
||||
res = {}
|
||||
for k, v in self.cfg.items():
|
||||
res.update(self._expand_matrix(k, v))
|
||||
|
||||
for k, v in self.cfg.items():
|
||||
deploy = v.get('deploy')
|
||||
if deploy:
|
||||
for item in _list(deploy):
|
||||
res['{}-{}'.format(k, item)] = v
|
||||
return res
|
||||
|
||||
@staticmethod
|
||||
def _expand_matrix(name, cfg): # type: (str, dict) -> dict
|
||||
"""
|
||||
Expand matrix into multi keys
|
||||
:param cfg: single rule dict
|
||||
:return:
|
||||
"""
|
||||
default = {name: cfg}
|
||||
if not cfg:
|
||||
return default
|
||||
matrices = cfg.pop('matrix', None)
|
||||
if not matrices:
|
||||
return default
|
||||
|
||||
res = {}
|
||||
for comb in product(*_list(matrices)):
|
||||
res.update(_format_nested_dict(default, comb))
|
||||
return res
|
||||
|
||||
def expand_rules(self): # type: () -> dict[str, dict[str, list]]
|
||||
res = defaultdict(lambda: defaultdict(set)) # type: dict[str, dict[str, set]]
|
||||
for k, v in self.cfg.items():
|
||||
for vk, vv in v.items():
|
||||
if vk in self.KEYWORDS:
|
||||
res[k][vk] = set(_list(vv))
|
||||
else:
|
||||
res[k][vk] = vv
|
||||
for key in self.KEYWORDS: # provide empty set for missing field
|
||||
if key not in res[k]:
|
||||
res[k][key] = set()
|
||||
|
||||
for k, v in self.cfg.items():
|
||||
if not v:
|
||||
continue
|
||||
if 'included_in' in v:
|
||||
for item in _list(v['included_in']):
|
||||
if 'labels' in v:
|
||||
res[item]['labels'].update(_list(v['labels']))
|
||||
if 'patterns' in v:
|
||||
for _pat in _list(v['patterns']):
|
||||
# Patterns must be pre-defined
|
||||
if '.patterns-{}'.format(_pat) not in self.rules_cfg:
|
||||
print('WARNING: pattern {} not exists'.format(_pat))
|
||||
continue
|
||||
res[item]['patterns'].add(_pat)
|
||||
|
||||
sorted_res = defaultdict(lambda: defaultdict(list)) # type: dict[str, dict[str, list]]
|
||||
for k, v in res.items():
|
||||
for vk, vv in v.items():
|
||||
sorted_res[k][vk] = sorted(vv)
|
||||
return sorted_res
|
||||
|
||||
def new_labels_titles_str(self): # type: () -> str
|
||||
_labels = set([])
|
||||
for k, v in self.cfg.items():
|
||||
if not v:
|
||||
continue # shouldn't be possible
|
||||
labels = v.get('labels')
|
||||
if not labels:
|
||||
continue
|
||||
_labels.update(_list(labels))
|
||||
labels = sorted(_labels)
|
||||
|
||||
res = ''
|
||||
res += '\n\n'.join([self._format_label(_label) for _label in labels])
|
||||
res += '\n\n'
|
||||
res += '\n\n'.join([self._format_title(_label) for _label in labels])
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
def _format_label(cls, label): # type: (str) -> str
|
||||
return cls.LABEL_TEMPLATE.format(label, cls.bot_label_str(label))
|
||||
|
||||
@staticmethod
|
||||
def bot_label_str(label): # type: (str) -> str
|
||||
return label.upper().replace('-', '_')
|
||||
|
||||
@classmethod
|
||||
def _format_title(cls, title): # type: (str) -> str
|
||||
return cls.TITLE_TEMPLATE.format(title)
|
||||
|
||||
def new_rules_str(self): # type: () -> str
|
||||
res = []
|
||||
for k, v in sorted(self.rules.items()):
|
||||
res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v)))
|
||||
return '\n\n'.join(res)
|
||||
|
||||
def _format_rule(self, name, cfg): # type: (str, dict) -> str
|
||||
_rules = []
|
||||
if name.endswith('-production'):
|
||||
_rules.append(self.RULE_PROD)
|
||||
else:
|
||||
if not name.endswith('-preview'):
|
||||
_rules.append(self.RULE_NORM)
|
||||
for label in cfg['labels']:
|
||||
_rules.append(self.RULE_LABEL_TEMPLATE.format(label))
|
||||
_rules.append(self.RULE_TITLE_TEMPLATE.format(label))
|
||||
for pattern in cfg['patterns']:
|
||||
if '.patterns-{}'.format(pattern) in self.rules_cfg:
|
||||
_rules.append(self.RULE_PATTERN_TEMPLATE.format(pattern))
|
||||
else:
|
||||
print('WARNING: pattern {} not exists'.format(pattern))
|
||||
return '\n'.join(_rules)
|
||||
|
||||
def update_rules_yml(self): # type: () -> bool
|
||||
with open(self.rules_yml) as fr:
|
||||
file_str = fr.read()
|
||||
|
||||
auto_generate_str = '\n{}\n\n{}\n'.format(self.new_labels_titles_str(), self.new_rules_str())
|
||||
rest, marker, old = file_str.partition(self.AUTO_GENERATE_MARKER)
|
||||
if old == auto_generate_str:
|
||||
return False
|
||||
else:
|
||||
print(self.rules_yml, 'has been modified. Please check')
|
||||
with open(self.rules_yml, 'w') as fw:
|
||||
fw.write(rest + marker + auto_generate_str)
|
||||
return True
|
||||
|
||||
|
||||
LABEL_COLOR = 'green'
|
||||
PATTERN_COLOR = 'cyan'
|
||||
RULE_COLOR = 'blue'
|
||||
|
||||
|
||||
def build_graph(rules_dict): # type: (dict[str, dict[str, list]]) -> pgv.AGraph
|
||||
graph = pgv.AGraph(directed=True, rankdir='LR', concentrate=True)
|
||||
|
||||
for k, v in rules_dict.items():
|
||||
if not v:
|
||||
continue
|
||||
included_in = v.get('included_in')
|
||||
if included_in:
|
||||
for item in _list(included_in):
|
||||
graph.add_node(k, color=RULE_COLOR)
|
||||
graph.add_node(item, color=RULE_COLOR)
|
||||
graph.add_edge(k, item, color=RULE_COLOR)
|
||||
labels = v.get('labels')
|
||||
if labels:
|
||||
for _label in labels:
|
||||
graph.add_node('label:{}'.format(_label), color=LABEL_COLOR)
|
||||
graph.add_edge('label:{}'.format(_label), k, color=LABEL_COLOR)
|
||||
patterns = v.get('patterns')
|
||||
if patterns:
|
||||
for _pat in patterns:
|
||||
graph.add_node('pattern:{}'.format(_pat), color=PATTERN_COLOR)
|
||||
graph.add_edge('pattern:{}'.format(_pat), k, color=PATTERN_COLOR)
|
||||
|
||||
return graph
|
||||
|
||||
|
||||
def output_graph(graph, output_path='output.png'): # type: (pgv.AGraph, str) -> None
|
||||
graph.layout('dot')
|
||||
if output_path.endswith('.png'):
|
||||
img_path = output_path
|
||||
else:
|
||||
img_path = os.path.join(output_path, 'output.png')
|
||||
graph.draw(img_path)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('rules_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'),
|
||||
help='rules.yml file path')
|
||||
parser.add_argument('dependencies_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'dependencies',
|
||||
'dependencies.yml'),
|
||||
help='dependencies.yml file path')
|
||||
parser.add_argument('--graph',
|
||||
help='Specify PNG image output path. Use this argument to generate dependency graph')
|
||||
args = parser.parse_args()
|
||||
|
||||
writer = RulesWriter(args.rules_yml, args.dependencies_yml)
|
||||
file_modified = writer.update_rules_yml()
|
||||
|
||||
if args.graph:
|
||||
dep_tree_graph = build_graph(writer.rules)
|
||||
output_graph(dep_tree_graph)
|
||||
|
||||
sys.exit(file_modified)
|
||||
@ -1,15 +1,15 @@
|
||||
.deploy_job_template:
|
||||
extends: .before_script_no_sync_submodule
|
||||
stage: deploy
|
||||
image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
|
||||
tags:
|
||||
- deploy
|
||||
dependencies: []
|
||||
|
||||
push_to_github:
|
||||
extends:
|
||||
- .deploy_job_template
|
||||
- .before_script_lesser
|
||||
- .rules:protected-no_label
|
||||
dependencies: []
|
||||
script:
|
||||
- add_github_ssh_keys
|
||||
- git remote remove github &>/dev/null || true
|
||||
@ -19,7 +19,6 @@ push_to_github:
|
||||
deploy_test_result:
|
||||
extends:
|
||||
- .deploy_job_template
|
||||
- .before_script_slim
|
||||
- .rules:ref:master-schedule-always
|
||||
image: $CI_DOCKER_REGISTRY/bot-env
|
||||
tags:
|
||||
@ -51,3 +50,24 @@ deploy_test_result:
|
||||
- echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
|
||||
# update test results
|
||||
- python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l CI -p ${CI_PROJECT_DIR}/TEST_LOGS ${CI_PROJECT_DIR}/${CI_COMMIT_SHA} --pipeline_url ${CI_PIPELINE_URL}
|
||||
|
||||
check_submodule_sync:
|
||||
extends:
|
||||
- .deploy_job_template
|
||||
- .rules:protected
|
||||
tags:
|
||||
- github_sync
|
||||
retry: 2
|
||||
variables:
|
||||
GIT_STRATEGY: clone
|
||||
SUBMODULES_TO_FETCH: "none"
|
||||
PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
|
||||
script:
|
||||
- git submodule deinit --force .
|
||||
# setting the default remote URL to the public one, to resolve relative location URLs
|
||||
- git config remote.origin.url ${PUBLIC_IDF_URL}
|
||||
# check if all submodules are correctly synced to public repository
|
||||
- git submodule init
|
||||
- git config --get-regexp '^submodule\..*\.url$' || true
|
||||
- git submodule update --recursive
|
||||
- echo "IDF was cloned from ${PUBLIC_IDF_URL} completely"
|
||||
|
||||
@ -1,7 +1,26 @@
|
||||
# stage: pre_check
|
||||
check_readme_links:
|
||||
extends:
|
||||
- .pre_check_job_template
|
||||
- .rules:build:docs
|
||||
tags: ["build", "amd64", "internet"]
|
||||
allow_failure: true
|
||||
script:
|
||||
- python ${IDF_PATH}/tools/ci/check_readme_links.py
|
||||
|
||||
check_docs_lang_sync:
|
||||
extends:
|
||||
- .pre_check_job_template
|
||||
- .rules:build:docs
|
||||
script:
|
||||
- cd docs
|
||||
- ./check_lang_folder_sync.sh
|
||||
|
||||
.build_docs_template:
|
||||
image: $ESP_IDF_DOC_ENV_IMAGE
|
||||
tags:
|
||||
- build_docs
|
||||
dependencies: []
|
||||
script:
|
||||
- cd docs
|
||||
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
|
||||
@ -9,37 +28,13 @@
|
||||
parallel:
|
||||
matrix:
|
||||
- DOCLANG: ["en", "zh_CN"]
|
||||
DOCTGT: ["esp32", "esp32s2", "esp32c3"]
|
||||
|
||||
# stage: pre_check
|
||||
check_readme_links:
|
||||
extends:
|
||||
- .pre_check_job_template
|
||||
- .rules:patterns:docs
|
||||
tags: ["internet"]
|
||||
allow_failure: true
|
||||
variables:
|
||||
PYTHON_VER: 3
|
||||
script:
|
||||
- python ${IDF_PATH}/tools/ci/check_readme_links.py
|
||||
|
||||
check_docs_lang_sync:
|
||||
extends:
|
||||
- .pre_check_job_template
|
||||
- .rules:patterns:docs
|
||||
variables:
|
||||
SUBMODULES_TO_FETCH: "none"
|
||||
script:
|
||||
- cd docs
|
||||
- ./check_lang_folder_sync.sh
|
||||
DOCTGT: ["esp32", "esp32s2"]
|
||||
|
||||
check_docs_gh_links:
|
||||
extends:
|
||||
- .pre_check_job_template
|
||||
- .build_docs_template
|
||||
- .rules:patterns:docs
|
||||
stage: pre_check
|
||||
variables:
|
||||
SUBMODULES_TO_FETCH: "none"
|
||||
- .rules:build:docs
|
||||
script:
|
||||
- cd docs
|
||||
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
|
||||
@ -49,11 +44,13 @@ check_docs_gh_links:
|
||||
.build_docs_build_stage_template:
|
||||
extends:
|
||||
- .build_docs_template
|
||||
- .rules:patterns:docs
|
||||
- .rules:build:docs
|
||||
stage: build
|
||||
needs:
|
||||
- check_docs_lang_sync
|
||||
- check_docs_gh_links
|
||||
- job: check_docs_lang_sync
|
||||
artifacts: false
|
||||
- job: check_docs_gh_links
|
||||
artifacts: false
|
||||
|
||||
build_docs_html:
|
||||
extends:
|
||||
@ -80,15 +77,16 @@ build_docs_pdf:
|
||||
|
||||
.deploy_docs_template:
|
||||
extends:
|
||||
- .before_script_lesser
|
||||
- .rules:patterns:docs
|
||||
- .before_script_no_sync_submodule
|
||||
image: $ESP_IDF_DOC_ENV_IMAGE
|
||||
stage: test_deploy
|
||||
tags:
|
||||
- deploy
|
||||
- shiny
|
||||
variables:
|
||||
DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
|
||||
PYTHONUNBUFFERED: 1
|
||||
dependencies: []
|
||||
script:
|
||||
- add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER
|
||||
- export GIT_VER=$(git describe --always)
|
||||
@ -98,8 +96,8 @@ build_docs_pdf:
|
||||
deploy_docs_preview:
|
||||
extends:
|
||||
- .deploy_docs_template
|
||||
- .rules:patterns:docs-preview
|
||||
stage: test_deploy
|
||||
- .rules:build:docs-preview
|
||||
dependencies: # set dependencies to null to avoid missing artifacts issue
|
||||
needs:
|
||||
- build_docs_html
|
||||
- build_docs_pdf
|
||||
@ -117,12 +115,14 @@ deploy_docs_production:
|
||||
# The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
|
||||
extends:
|
||||
- .deploy_docs_template
|
||||
- .rules:protected-no_label
|
||||
- .rules:build:docs-production
|
||||
stage: post_deploy
|
||||
dependencies: # set dependencies to null to avoid missing artifacts issue
|
||||
needs: # ensure runs after push_to_github succeeded
|
||||
- build_docs_html
|
||||
- build_docs_pdf
|
||||
- push_to_github
|
||||
- job: push_to_github
|
||||
artifacts: false
|
||||
variables:
|
||||
TYPE: "preview"
|
||||
DOCS_DEPLOY_PRIVATEKEY: "$DOCS_PROD_DEPLOY_KEY"
|
||||
@ -144,7 +144,6 @@ check_doc_links:
|
||||
- docs/_build/*/*/linkcheck/*.txt
|
||||
expire_in: 1 week
|
||||
allow_failure: true
|
||||
dependencies: []
|
||||
script:
|
||||
- cd docs
|
||||
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
|
||||
|
||||
@ -1,30 +1,11 @@
|
||||
.host_test_template:
|
||||
extends: .rules:labels:host_test
|
||||
extends: .rules:test:host_test
|
||||
stage: host_test
|
||||
image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
|
||||
tags:
|
||||
- host_test
|
||||
dependencies: []
|
||||
|
||||
.host_fuzzer_test_template:
|
||||
extends:
|
||||
- .host_test_template
|
||||
- .rules:labels:fuzzer_test-weekend_test-only
|
||||
image: $CI_DOCKER_REGISTRY/afl-fuzzer-test
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- ${FUZZER_TEST_DIR}/out/crashes
|
||||
- ${FUZZER_TEST_DIR}/fuzz_output.log
|
||||
expire_in: 1 week
|
||||
script:
|
||||
- export AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 && export AFL_SKIP_CPUFREQ=1
|
||||
- cd ${FUZZER_TEST_DIR}
|
||||
# run AFL fuzzer for one hour
|
||||
- ( ( make ${FUZZER_PARAMS} fuzz | tee fuzz_output.log | grep -v '\(Fuzzing test case\|Entering queue cycle\)' ) || pkill sleep ) &
|
||||
- ( sleep 3600 || mkdir -p out/crashes/env_failed ) && pkill afl-fuz
|
||||
# check no crashes found
|
||||
- test -z "$(ls out/crashes/)" || exit 1
|
||||
needs: [] # run host_test jobs immediately
|
||||
|
||||
test_nvs_on_host:
|
||||
extends: .host_test_template
|
||||
@ -35,7 +16,7 @@ test_nvs_on_host:
|
||||
test_nvs_coverage:
|
||||
extends:
|
||||
- .host_test_template
|
||||
- .rules:labels:nvs_coverage-only
|
||||
- .rules:labels:nvs_coverage
|
||||
artifacts:
|
||||
paths:
|
||||
- components/nvs_flash/test_nvs_host/coverage_report
|
||||
@ -79,6 +60,26 @@ test_ldgen_on_host:
|
||||
variables:
|
||||
LC_ALL: C.UTF-8
|
||||
|
||||
.host_fuzzer_test_template:
|
||||
extends:
|
||||
- .host_test_template
|
||||
- .rules:labels:fuzzer_test-weekend_test
|
||||
image: $CI_DOCKER_REGISTRY/afl-fuzzer-test
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- ${FUZZER_TEST_DIR}/out/crashes
|
||||
- ${FUZZER_TEST_DIR}/fuzz_output.log
|
||||
expire_in: 1 week
|
||||
script:
|
||||
- export AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 && export AFL_SKIP_CPUFREQ=1
|
||||
- cd ${FUZZER_TEST_DIR}
|
||||
# run AFL fuzzer for one hour
|
||||
- ( ( make ${FUZZER_PARAMS} fuzz | tee fuzz_output.log | grep -v '\(Fuzzing test case\|Entering queue cycle\)' ) || pkill sleep ) &
|
||||
- ( sleep 3600 || mkdir -p out/crashes/env_failed ) && pkill afl-fuz
|
||||
# check no crashes found
|
||||
- test -z "$(ls out/crashes/)" || exit 1
|
||||
|
||||
test_mdns_fuzzer_on_host:
|
||||
extends: .host_fuzzer_test_template
|
||||
variables:
|
||||
|
||||
@ -1,73 +0,0 @@
|
||||
# copy from .gitlab-ci.yml as anchor is not global
|
||||
.show_submodule_urls: &show_submodule_urls |
|
||||
git config --get-regexp '^submodule\..*\.url$' || true
|
||||
|
||||
.post_check_base_template:
|
||||
stage: post_check
|
||||
image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
|
||||
tags:
|
||||
- host_test
|
||||
dependencies: []
|
||||
|
||||
.post_check_job_template:
|
||||
extends:
|
||||
- .post_check_base_template
|
||||
- .before_script_lesser_nofilter
|
||||
|
||||
.post_check_job_template_with_filter:
|
||||
extends:
|
||||
- .post_check_base_template
|
||||
- .before_script_lesser
|
||||
|
||||
check_submodule_sync:
|
||||
extends:
|
||||
- .before_script_slim
|
||||
- .post_check_job_template
|
||||
tags:
|
||||
- github_sync
|
||||
retry: 2
|
||||
variables:
|
||||
GIT_STRATEGY: clone
|
||||
SUBMODULES_TO_FETCH: "none"
|
||||
PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
|
||||
script:
|
||||
- git submodule deinit --force .
|
||||
# setting the default remote URL to the public one, to resolve relative location URLs
|
||||
- git config remote.origin.url ${PUBLIC_IDF_URL}
|
||||
# check if all submodules are correctly synced to public repository
|
||||
- git submodule init
|
||||
- *show_submodule_urls
|
||||
- git submodule update --recursive
|
||||
- echo "IDF was cloned from ${PUBLIC_IDF_URL} completely"
|
||||
|
||||
check_ut_cmake_make:
|
||||
extends:
|
||||
- .post_check_job_template_with_filter
|
||||
- .rules:dev
|
||||
tags:
|
||||
- build
|
||||
script:
|
||||
- tools/ci/check_ut_cmake_make.sh
|
||||
|
||||
check_artifacts_expire_time:
|
||||
extends: .post_check_job_template
|
||||
script:
|
||||
# check if we have set expire time for all artifacts
|
||||
- python tools/ci/check_artifacts_expire_time.py
|
||||
|
||||
check_pipeline_triggered_by_label:
|
||||
extends:
|
||||
- .post_check_job_template
|
||||
- .rules:dev
|
||||
script:
|
||||
# If the pipeline is triggered with label, the pipeline will only succeeded if "regular_test" label is added.
|
||||
# We want to make sure some jobs are always executed to detect regression.
|
||||
- test "$BOT_LABEL_REGULAR_TEST" = "true" || { echo "CI can only pass if 'regular_test' label is included"; exit -1; }
|
||||
|
||||
check_commit_msg:
|
||||
extends: .post_check_job_template
|
||||
script:
|
||||
- git status
|
||||
- git log -n10 --oneline
|
||||
# commit start with "WIP: " need to be squashed before merge
|
||||
- 'git log --pretty=%s master.. -- | grep "^WIP: " && exit 1 || exit 0'
|
||||
@ -8,16 +8,10 @@
|
||||
.pre_check_job_template:
|
||||
extends:
|
||||
- .pre_check_base_template
|
||||
- .before_script_lesser_nofilter
|
||||
|
||||
.pre_check_job_template_with_filter:
|
||||
extends:
|
||||
- .pre_check_base_template
|
||||
- .before_script_lesser
|
||||
- .before_script_no_sync_submodule
|
||||
|
||||
.check_pre_commit_template:
|
||||
extends: .pre_check_job_template
|
||||
stage: pre_check
|
||||
image: "$CI_DOCKER_REGISTRY/esp-idf-pre-commit:1"
|
||||
before_script:
|
||||
- source tools/ci/utils.sh
|
||||
@ -48,14 +42,12 @@ check_version:
|
||||
- tools/ci/check_idf_version.sh
|
||||
|
||||
check_examples_cmake_make:
|
||||
extends:
|
||||
- .pre_check_job_template_with_filter
|
||||
- .rules:dev
|
||||
extends: .pre_check_job_template
|
||||
script:
|
||||
- python ${IDF_PATH}/tools/ci/check_examples_cmake_make.py
|
||||
- python ${IDF_PATH}/tools/ci/check_examples_cmake_make.py
|
||||
|
||||
check_rom_api_header:
|
||||
extends: .pre_check_job_template_with_filter
|
||||
extends: .pre_check_job_template
|
||||
script:
|
||||
- tools/ci/check_examples_rom_header.sh
|
||||
- tools/ci/check_rom_apis.sh
|
||||
@ -72,8 +64,8 @@ check_python_style:
|
||||
script:
|
||||
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
|
||||
|
||||
check_kconfigs:
|
||||
extends: .pre_check_job_template_with_filter
|
||||
test_check_kconfigs:
|
||||
extends: .pre_check_job_template
|
||||
artifacts:
|
||||
when: on_failure
|
||||
paths:
|
||||
@ -86,7 +78,6 @@ check_kconfigs:
|
||||
expire_in: 1 week
|
||||
script:
|
||||
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/ci/test_check_kconfigs.py
|
||||
- ${IDF_PATH}/tools/ci/check_kconfigs.py
|
||||
|
||||
check_wifi_lib_md5:
|
||||
extends: .pre_check_base_template
|
||||
@ -101,25 +92,19 @@ check_wifi_lib_md5:
|
||||
check_public_headers:
|
||||
extends:
|
||||
- .pre_check_base_template
|
||||
- .rules:labels:build
|
||||
- .rules:build
|
||||
tags:
|
||||
- build
|
||||
script:
|
||||
- python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32-elf-
|
||||
|
||||
.scan_build_tests:
|
||||
stage: pre_check
|
||||
scan_tests:
|
||||
extends:
|
||||
- .pre_check_base_template
|
||||
- .rules:test:target_test
|
||||
image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
|
||||
tags:
|
||||
- assign_test
|
||||
variables:
|
||||
CI_SCAN_TESTS_PY: ${CI_PROJECT_DIR}/tools/ci/python_packages/ttfw_idf/CIScanTests.py
|
||||
TEST_CONFIG_FILE: ${CI_PROJECT_DIR}/tools/ci/config/target-test.yml
|
||||
|
||||
scan_tests:
|
||||
extends:
|
||||
- .scan_build_tests
|
||||
- .rules:build_tests:target_test-weekend_test
|
||||
artifacts:
|
||||
paths:
|
||||
- $EXAMPLE_TEST_OUTPUT_DIR
|
||||
@ -131,23 +116,44 @@ scan_tests:
|
||||
TEST_APPS_TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps
|
||||
TEST_APPS_OUTPUT_DIR: ${CI_PROJECT_DIR}/tools/test_apps/test_configs
|
||||
COMPONENT_UT_OUTPUT_DIR: ${CI_PROJECT_DIR}/component_ut/test_configs
|
||||
PYTHON_VER: 3
|
||||
CI_SCAN_TESTS_PY: ${CI_PROJECT_DIR}/tools/ci/python_packages/ttfw_idf/CIScanTests.py
|
||||
script:
|
||||
- set_component_ut_vars
|
||||
- python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b make --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
|
||||
- python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b cmake --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
|
||||
- python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR
|
||||
- python $CI_SCAN_TESTS_PY component_ut $COMPONENT_UT_DIRS --exclude $COMPONENT_UT_EXCLUDES -c $TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR
|
||||
- run_cmd python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b make --exclude examples/build_system/idf_as_lib -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
|
||||
- run_cmd python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b cmake --exclude examples/build_system/idf_as_lib -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
|
||||
- run_cmd python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR
|
||||
- run_cmd python $CI_SCAN_TESTS_PY component_ut $COMPONENT_UT_DIRS --exclude $COMPONENT_UT_EXCLUDES -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR
|
||||
|
||||
# For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update
|
||||
# the version returned by 'git describe'
|
||||
check_version_tag:
|
||||
extends:
|
||||
- .pre_check_job_template
|
||||
- .rules:tag:release-no_label
|
||||
- .rules:tag:release
|
||||
script:
|
||||
- (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1)
|
||||
|
||||
check_ut_cmake_make:
|
||||
extends: .pre_check_job_template
|
||||
tags:
|
||||
- build
|
||||
script:
|
||||
- tools/ci/check_ut_cmake_make.sh
|
||||
|
||||
check_artifacts_expire_time:
|
||||
extends: .pre_check_job_template
|
||||
script:
|
||||
# check if we have set expire time for all artifacts
|
||||
- python tools/ci/check_artifacts_expire_time.py
|
||||
|
||||
check_commit_msg:
|
||||
extends: .pre_check_job_template
|
||||
script:
|
||||
- git status
|
||||
- git log -n10 --oneline
|
||||
# commit start with "WIP: " need to be squashed before merge
|
||||
- 'git log --pretty=%s master.. -- | grep "^WIP: " && exit 1 || exit 0'
|
||||
|
||||
check_tools_file_patterns:
|
||||
extends: .pre_check_job_template
|
||||
image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
|
||||
|
||||
1219
.gitlab/ci/rules.yml
1219
.gitlab/ci/rules.yml
File diff suppressed because it is too large
Load Diff
@ -1,4 +1,5 @@
|
||||
.clang_tidy_check_template:
|
||||
# pre_check stage
|
||||
clang_tidy_check:
|
||||
extends:
|
||||
- .pre_check_base_template
|
||||
- .rules:patterns:clang_tidy
|
||||
@ -16,24 +17,6 @@
|
||||
- export TARGET_BRANCH=${BOT_CUSTOMIZED_REVISION-}
|
||||
- ./analyze.sh $IDF_PATH/examples/get-started/hello_world/ $IDF_PATH/tools/ci/static-analysis-rules.yml $IDF_PATH/output.xml
|
||||
|
||||
# pre_check stage
|
||||
clang_tidy_check:
|
||||
extends: .clang_tidy_check_template
|
||||
variables:
|
||||
BOT_NEEDS_TRIGGER_BY_NAME: 1
|
||||
TRIGGERED_RELATIVE: 1
|
||||
|
||||
clang_tidy_check_regular:
|
||||
extends:
|
||||
- .clang_tidy_check_template
|
||||
- .rules:patterns:clang_tidy-preview
|
||||
|
||||
clang_tidy_check_all:
|
||||
extends: .clang_tidy_check_template
|
||||
variables:
|
||||
BOT_NEEDS_TRIGGER_BY_NAME: 1
|
||||
TRIGGERED_ABSOLUTE: 1
|
||||
|
||||
# build stage
|
||||
# Sonarqube related jobs put here for this reason:
|
||||
# Here we have two jobs. code_quality_check and code_quality_report.
|
||||
@ -69,7 +52,7 @@ clang_tidy_check_all:
|
||||
tags:
|
||||
- host_test
|
||||
dependencies: # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here.
|
||||
- clang_tidy_check_regular
|
||||
- clang_tidy_check
|
||||
|
||||
code_quality_check:
|
||||
extends:
|
||||
@ -104,7 +87,7 @@ code_quality_check:
|
||||
code_quality_report:
|
||||
extends:
|
||||
- .sonar_scan_template
|
||||
- .rules:protected-schedule
|
||||
- .rules:protected
|
||||
script:
|
||||
- sonar-scanner
|
||||
-Dsonar.branch.name=$CI_COMMIT_REF_NAME
|
||||
@ -124,10 +107,12 @@ code_quality_report:
|
||||
-Dsonar.sources=$CI_PROJECT_DIR
|
||||
|
||||
# deploy stage
|
||||
.clang_tidy_deploy_template:
|
||||
clang_tidy_deploy:
|
||||
extends:
|
||||
- .deploy_job_template
|
||||
- .rules:patterns:clang_tidy
|
||||
needs:
|
||||
- clang_tidy_check
|
||||
tags:
|
||||
- deploy
|
||||
- shiny
|
||||
@ -144,20 +129,3 @@ code_quality_report:
|
||||
# add link to view the report
|
||||
- echo "[static analysis][clang tidy] $CI_DOCKER_REGISTRY/static_analysis/esp-idf/clang-tidy/${GIT_VER}/index.html"
|
||||
- test ! -e ${GIT_VER}/FAILED_RULES || { echo 'Failed static analysis rules!'; cat ${GIT_VER}/FAILED_RULES; exit 1; }
|
||||
|
||||
clang_tidy_deploy:
|
||||
extends: .clang_tidy_deploy_template
|
||||
# Override default stage to happen before the post_check
|
||||
stage: test_deploy
|
||||
needs:
|
||||
- clang_tidy_check
|
||||
- clang_tidy_check_all
|
||||
variables:
|
||||
BOT_NEEDS_TRIGGER_BY_NAME: 1
|
||||
|
||||
clang_tidy_deploy_regular:
|
||||
extends:
|
||||
- .clang_tidy_deploy_template
|
||||
- .rules:patterns:clang_tidy-preview
|
||||
needs:
|
||||
- clang_tidy_check_regular
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,63 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# internal use only
|
||||
# called by CI jobs to determine if it need to be executed
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
RE_FILTER_PATTERN = re.compile(r'^r"(.+)?"$')
|
||||
|
||||
RE_TYPE = type(re.compile('', 0))
|
||||
|
||||
|
||||
def parse_filter(filter_name):
|
||||
filter_raw = os.getenv(filter_name)
|
||||
filters = []
|
||||
if filter_raw:
|
||||
filter_data = json.loads(filter_raw)
|
||||
for _filter in filter_data:
|
||||
match = RE_FILTER_PATTERN.search(_filter)
|
||||
if match:
|
||||
filters.append(re.compile(match.group(1)))
|
||||
else:
|
||||
filters.append(_filter)
|
||||
return filters
|
||||
|
||||
|
||||
def process_filter(execute_by_default, filter_name, ci_name):
|
||||
execute = execute_by_default
|
||||
|
||||
# bot message is case insensitive (processed with lower case). so we also convert ci_name to lower case.
|
||||
ci_name = ci_name.lower()
|
||||
|
||||
filter_list = parse_filter(filter_name)
|
||||
|
||||
for _filter in filter_list:
|
||||
if isinstance(_filter, RE_TYPE):
|
||||
match = _filter.search(ci_name) is not None
|
||||
else:
|
||||
match = _filter == ci_name
|
||||
|
||||
if match:
|
||||
execute = True
|
||||
break
|
||||
else:
|
||||
execute = False
|
||||
return execute
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
execute_by_default = True
|
||||
if os.getenv('BOT_NEEDS_TRIGGER_BY_NAME', '0') == '1':
|
||||
execute_by_default = False
|
||||
|
||||
need_to_execute = process_filter(True, 'BOT_STAGE_FILTER', os.getenv('CI_JOB_STAGE')) and process_filter(execute_by_default,
|
||||
'BOT_JOB_FILTER', os.getenv('CI_JOB_NAME'))
|
||||
if need_to_execute:
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("Skip this job as it doesn't fit @bot's filter")
|
||||
sys.exit(-1)
|
||||
@ -24,7 +24,6 @@ import yaml
|
||||
from idf_ci_utils import IDF_PATH, get_git_files, magic_check, magic_check_bytes, translate
|
||||
|
||||
# Monkey patch starts
|
||||
|
||||
# glob.glob will ignore all files starts with ``.``
|
||||
# don't ignore them here
|
||||
# need to keep the same argument as glob._ishidden
|
||||
@ -83,13 +82,13 @@ if __name__ == '__main__':
|
||||
not_included_files, dup_patterns = check(args.pattern_yml, args.exclude_list)
|
||||
if not_included_files:
|
||||
print('Missing Files: (please add to tools/ci/exclude_check_tools_files.txt')
|
||||
for f in not_included_files:
|
||||
print(f)
|
||||
for file in not_included_files:
|
||||
print(file)
|
||||
res = 1
|
||||
if dup_patterns:
|
||||
print('Duplicated Patterns: (please check .gitlab/ci/rules.yml and tools/ci/exclude_check_tools_files.txt')
|
||||
for pat in dup_patterns:
|
||||
print(pat)
|
||||
for pattern in dup_patterns:
|
||||
print(pattern)
|
||||
res = 1
|
||||
|
||||
sys.exit(res)
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
.gitlab/ci/dependencies/generate_rules.py
|
||||
components/app_update/otatool.py
|
||||
components/efuse/efuse_table_gen.py
|
||||
components/efuse/test_efuse_host/efuse_tests.py
|
||||
@ -33,7 +34,6 @@ install.fish
|
||||
install.sh
|
||||
tools/build_apps.py
|
||||
tools/check_python_dependencies.py
|
||||
tools/ci/apply_bot_filter.py
|
||||
tools/ci/build_template_app.sh
|
||||
tools/ci/check_build_warnings.py
|
||||
tools/ci/check_callgraph.py
|
||||
@ -46,6 +46,7 @@ tools/ci/check_idf_version.sh
|
||||
tools/ci/check_kconfigs.py
|
||||
tools/ci/check_readme_links.py
|
||||
tools/ci/check_rom_apis.sh
|
||||
tools/ci/check_rules_yml.py
|
||||
tools/ci/check_tools_files_patterns.py
|
||||
tools/ci/check_ut_cmake_make.sh
|
||||
tools/ci/checkout_project_ref.py
|
||||
|
||||
@ -1,10 +1,5 @@
|
||||
# Modified from https://gitlab.com/gitlab-org/gitlab/-/blob/master/scripts/utils.sh
|
||||
|
||||
# before each job, we need to check if this job is filtered by bot stage/job filter
|
||||
function apply_bot_filter() {
|
||||
python "${IDF_PATH}"/tools/ci/apply_bot_filter.py || exit 0
|
||||
}
|
||||
|
||||
function add_ssh_keys() {
|
||||
local key_string="${1}"
|
||||
mkdir -p ~/.ssh
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user