From c9971bea592fb6254d5302727096662776e28f51 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 27 Aug 2024 14:15:16 +0200 Subject: [PATCH] ci: support diff manifest sha with the base commit --- .gitignore | 1 + .gitlab/ci/build.yml | 2 ++ .gitlab/ci/pre_check.yml | 20 +++++++++++++++++++ .gitlab/ci/rules.yml | 4 ++++ .../scripts/generate_build_child_pipeline.py | 10 ++++++++++ tools/ci/idf_pytest/script.py | 4 ++++ 6 files changed, 41 insertions(+) diff --git a/.gitignore b/.gitignore index f70980b703..313692055c 100644 --- a/.gitignore +++ b/.gitignore @@ -102,6 +102,7 @@ pytest_embedded_log/ list_job*.txt size_info*.txt XUNIT_RESULT*.xml +.manifest_sha # clang config (for LSP) .clangd diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index ef04ea5ab4..b2e805f3c9 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -264,6 +264,8 @@ generate_build_child_pipeline: dependencies: # set dependencies to null to avoid missing artifacts issue needs: - pipeline_variables + - job: baseline_manifest_sha + optional: true artifacts: paths: - build_child_pipeline.yml diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index 5e139646a8..4f55b737da 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -169,3 +169,23 @@ pipeline_variables: - pipeline.env expire_in: 1 week when: always + +baseline_manifest_sha: + extends: + - .pre_check_template + - .rules:dev-push + tags: [fast_run, shiny] + script: + - | + if [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then + git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 + git checkout FETCH_HEAD + idf-build-apps dump-manifest-sha \ + --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \ + --output .manifest_sha + fi + artifacts: + paths: + - .manifest_sha + expire_in: 1 week + when: always diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index a9557cf916..b68f6e6056 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -217,6 +217,10 @@ rules: - <<: *if-tag-release +.rules:dev-push: + rules: + - <<: *if-dev-push + # Do not upload caches on dev branches by default .rules:upload-python-cache: rules: diff --git a/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py index 7c8bf1d4d1..5941ce4dcd 100644 --- a/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py +++ b/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py @@ -89,6 +89,7 @@ def main(arguments: argparse.Namespace) -> None: marker_expr='not host_test', config_rules_str=DEFAULT_CONFIG_RULES_STR, extra_default_build_targets=extra_default_build_targets, + compare_manifest_sha_filepath=arguments.compare_manifest_sha_filepath, modified_components=arguments.modified_components, modified_files=arguments.modified_files, ignore_app_dependencies_filepatterns=arguments.ignore_app_dependencies_filepatterns, @@ -165,6 +166,11 @@ if __name__ == '__main__': default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), help='default build test rules config file', ) + parser.add_argument( + '--compare-manifest-sha-filepath', + default=os.path.join(IDF_PATH, '.manifest_sha'), + help='Path to the recorded manifest sha file generated by `idf-build-apps dump-manifest-sha`' + ) parser.add_argument( '--modified-components', type=_separate_str_to_list, @@ -237,4 +243,8 @@ if __name__ == '__main__': # setting default values args.ignore_app_dependencies_filepatterns = DEFAULT_FULL_BUILD_TEST_FILEPATTERNS + if not os.path.isfile(args.compare_manifest_sha_filepath): + # ignore if the file does not exist + args.compare_manifest_sha_filepath = None + main(args) diff --git a/tools/ci/idf_pytest/script.py b/tools/ci/idf_pytest/script.py index eed0609e0f..0a2770bf5e 100644 --- a/tools/ci/idf_pytest/script.py +++ b/tools/ci/idf_pytest/script.py @@ -128,6 +128,7 @@ def get_all_apps( config_rules_str: t.Optional[t.List[str]] = None, preserve_all: bool = False, extra_default_build_targets: t.Optional[t.List[str]] = None, + compare_manifest_sha_filepath: t.Optional[str] = None, modified_components: t.Optional[t.List[str]] = None, modified_files: t.Optional[t.List[str]] = None, ignore_app_dependencies_filepatterns: t.Optional[t.List[str]] = None, @@ -142,8 +143,10 @@ def get_all_apps( :param config_rules_str: config rules string :param preserve_all: preserve all apps :param extra_default_build_targets: extra default build targets + :param compare_manifest_sha_filepath: check manifest sha filepath :param modified_components: modified components :param modified_files: modified files + :param ignore_app_dependencies_components: ignore app dependencies components :param ignore_app_dependencies_filepatterns: ignore app dependencies filepatterns :return: tuple of test-required apps and non-test-related apps """ @@ -161,6 +164,7 @@ def get_all_apps( size_json_filename='size.json', check_warnings=True, manifest_rootpath=IDF_PATH, + compare_manifest_sha_filepath=compare_manifest_sha_filepath, manifest_files=get_all_manifest_files(), default_build_targets=SUPPORTED_TARGETS + (extra_default_build_targets or []), modified_components=modified_components,