From 151d8cdcd2ceb1674319f50fcf2544fedb227658 Mon Sep 17 00:00:00 2001 From: Chen Yudong Date: Thu, 6 Jan 2022 23:21:21 +0800 Subject: [PATCH] ci: split assign tests --- .gitlab/ci/assign-test.yml | 121 +++++++++++---- .gitlab/ci/dependencies/dependencies.yml | 19 ++- .gitlab/ci/rules.yml | 145 +++++++++++------- .gitlab/ci/target-test.yml | 14 +- .../nvs_compatible_test_.yml | 0 5 files changed, 199 insertions(+), 100 deletions(-) rename components/idf_test/integration_test/{CIConfigs => test_configs}/nvs_compatible_test_.yml (100%) diff --git a/.gitlab/ci/assign-test.yml b/.gitlab/ci/assign-test.yml index 1471fb7156..cabe2f8578 100644 --- a/.gitlab/ci/assign-test.yml +++ b/.gitlab/ci/assign-test.yml @@ -1,48 +1,107 @@ -assign_test: - extends: .rules:test:any_test - tags: - - assign_test +.assign_test_template: image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG stage: assign_test - # gitlab ci do not support match job with RegEx or wildcard now in dependencies. - # we have a lot build example jobs. now we don't use dependencies, just download all artifacts of build stage. - dependencies: # Here is not a hard dependency relationship, could be skipped. so we do not use "needs" here. - - build_ssc_esp32 - - build_esp_idf_tests_cmake_esp32 - - build_esp_idf_tests_cmake_esp32s2 - - build_esp_idf_tests_cmake_esp32s3 - - build_esp_idf_tests_cmake_esp32c3 + tags: + - assign_test variables: - SUBMODULES_TO_FETCH: "components/esptool_py/esptool" - EXAMPLE_TEST_DIR: "${CI_PROJECT_DIR}/examples" - CUSTOM_TEST_DIR: "${CI_PROJECT_DIR}/tools/test_apps" - UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test" - INTEGRATION_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs" + SUBMODULES_TO_FETCH: components/esptool_py/esptool + artifacts: + paths: + - ${TEST_DIR}/test_configs + - ${BUILD_DIR}/artifact_index.json + when: always + expire_in: 1 week + script: + - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs + +assign_example_test: + extends: + - .assign_test_template + - .rules:build:example_test + needs: + - job: build_examples_cmake_esp32 + artifacts: false + optional: true + - job: build_examples_cmake_esp32s2 + artifacts: false + optional: true + - job: build_examples_cmake_esp32c3 + artifacts: false + optional: true + - job: build_examples_cmake_esp32s3 + artifacts: false + optional: true + variables: + TEST_TYPE: example_test + TEST_DIR: ${CI_PROJECT_DIR}/examples + BUILD_DIR: ${CI_PROJECT_DIR}/build_examples + +assign_custom_test: + extends: + - .assign_test_template + - .rules:build:custom_test + needs: + - job: build_test_apps_esp32 + artifacts: false + optional: true + - job: build_test_apps_esp32s2 + artifacts: false + optional: true + - job: build_test_apps_esp32c3 + artifacts: false + optional: true + - job: build_test_apps_esp32s3 + artifacts: false + optional: true + variables: + TEST_TYPE: custom_test + TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps + BUILD_DIR: ${CI_PROJECT_DIR}/build_test_apps + +assign_unit_test: + extends: + - .assign_test_template + - .rules:build:unit_test + needs: + - job: build_esp_idf_tests_cmake_esp32 + optional: true + - job: build_esp_idf_tests_cmake_esp32s2 + optional: true + - job: build_esp_idf_tests_cmake_esp32c3 + optional: true + - job: build_esp_idf_tests_cmake_esp32s3 + optional: true + variables: + TEST_TYPE: unit_test + TEST_DIR: ${CI_PROJECT_DIR}/components/idf_test/unit_test + BUILD_DIR: ${CI_PROJECT_DIR}/tools/unit-test-app/builds + script: + - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs + +assign_integration_test: + extends: + - .assign_test_template + - .rules:test:integration_test + needs: + - build_ssc_esp32 + artifacts: + paths: + - $TEST_DIR/test_configs + variables: + TEST_DIR: ${CI_PROJECT_DIR}/components/idf_test/integration_test + BUILD_DIR: ${CI_PROJECT_DIR}/SSC/ssc_bin INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles" ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py" PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages - # auto_test_script only supports python 3.7.x PYTHON_VER: 3.7.7 - artifacts: - paths: - - components/idf_test/*/CIConfigs - - $EXAMPLE_TEST_DIR/test_configs - - $CUSTOM_TEST_DIR/test_configs - - build_examples/artifact_index.json - - build_test_apps/artifact_index.json - - tools/unit-test-app/builds/artifact_index.json - expire_in: 1 week script: - - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py example_test $EXAMPLE_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_DIR/test_configs - - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py custom_test $CUSTOM_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $CUSTOM_TEST_DIR/test_configs - - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py unit_test $UNIT_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $UNIT_TEST_DIR/CIConfigs # clone test script to assign tests # can not retry if downing git lfs files failed, so using empty_branch first. - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH} - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only # assign integration test cases - - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_CONFIG_OUTPUT_PATH + - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b ${BUILD_DIR} -o $TEST_DIR/test_configs update_test_cases: extends: .rules:ref:master-schedule diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index 8c6bef2a08..41caf67012 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -63,6 +63,8 @@ patterns: - build_components - build_system + included_in: + - "build:{0}" build:integration_test: labels: @@ -85,10 +87,10 @@ build:integration_test: - "{0}" - "build-{0}" included_in: # Parent rules + - "build:{0}" - "build:{0}-{1}" - build:target_test - test:target_test - - test:any_test # ------------- # Special Cases @@ -105,29 +107,26 @@ build:integration_test: - component_ut - "build-component_ut-{0}" included_in: + - build:component_ut - "build:component_ut-{0}" - build:target_test - test:target_test - - test:any_test "test:integration_test": labels: - - "integration_test" + - integration_test patterns: - - "integration_test" + - integration_test included_in: - - "build:integration_test" + - build:integration_test - build:target_test - test:target_test - - test:any_test "test:host_test": labels: - host_test patterns: - host_test - included_in: - - test:any_test "test:submodule": labels: @@ -142,17 +141,17 @@ build:integration_test: labels: - iperf_stress_test included_in: + - build:example_test - build:example_test-esp32 - build:target_test - - test:any_test "labels:weekend_test": # custom test labels: - weekend_test included_in: + - build:custom_test - build:custom_test-esp32 - build:target_test - - test:any_test "labels:nvs_coverage": # host_test labels: diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 482541af7d..72d5bca968 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -403,6 +403,33 @@ - <<: *if-dev-push changes: *patterns-build_system +.rules:build:component_ut: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build + - <<: *if-label-component_ut + - <<: *if-label-component_ut_esp32 + - <<: *if-label-component_ut_esp32c2 + - <<: *if-label-component_ut_esp32c3 + - <<: *if-label-component_ut_esp32h2 + - <<: *if-label-component_ut_esp32s2 + - <<: *if-label-component_ut_esp32s3 + - <<: *if-label-unit_test + - <<: *if-label-unit_test_esp32 + - <<: *if-label-unit_test_esp32c2 + - <<: *if-label-unit_test_esp32c3 + - <<: *if-label-unit_test_esp32h2 + - <<: *if-label-unit_test_esp32s2 + - <<: *if-label-unit_test_esp32s3 + - <<: *if-dev-push + changes: *patterns-build_components + - <<: *if-dev-push + changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-component_ut + .rules:build:component_ut-esp32: rules: - <<: *if-revert-branch @@ -505,6 +532,27 @@ - <<: *if-dev-push changes: *patterns-component_ut +.rules:build:custom_test: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build + - <<: *if-label-custom_test + - <<: *if-label-custom_test_esp32 + - <<: *if-label-custom_test_esp32c2 + - <<: *if-label-custom_test_esp32c3 + - <<: *if-label-custom_test_esp32h2 + - <<: *if-label-custom_test_esp32s2 + - <<: *if-label-custom_test_esp32s3 + - <<: *if-label-weekend_test + - <<: *if-dev-push + changes: *patterns-build_components + - <<: *if-dev-push + changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-custom_test + .rules:build:custom_test-esp32: rules: - <<: *if-revert-branch @@ -606,6 +654,29 @@ - <<: *if-dev-push changes: *patterns-docker +.rules:build:example_test: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build + - <<: *if-label-example_test + - <<: *if-label-example_test_esp32 + - <<: *if-label-example_test_esp32c2 + - <<: *if-label-example_test_esp32c3 + - <<: *if-label-example_test_esp32h2 + - <<: *if-label-example_test_esp32s2 + - <<: *if-label-example_test_esp32s3 + - <<: *if-label-iperf_stress_test + - <<: *if-dev-push + changes: *patterns-build-example_test + - <<: *if-dev-push + changes: *patterns-build_components + - <<: *if-dev-push + changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-example_test + .rules:build:example_test-esp32: rules: - <<: *if-revert-branch @@ -788,6 +859,26 @@ - <<: *if-dev-push changes: *patterns-unit_test +.rules:build:unit_test: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build + - <<: *if-label-unit_test + - <<: *if-label-unit_test_esp32 + - <<: *if-label-unit_test_esp32c2 + - <<: *if-label-unit_test_esp32c3 + - <<: *if-label-unit_test_esp32h2 + - <<: *if-label-unit_test_esp32s2 + - <<: *if-label-unit_test_esp32s3 + - <<: *if-dev-push + changes: *patterns-build_components + - <<: *if-dev-push + changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-unit_test + .rules:build:unit_test-esp32: rules: - <<: *if-revert-branch @@ -915,60 +1006,6 @@ when: never - <<: *if-label-weekend_test -.rules:test:any_test: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32 - - <<: *if-label-component_ut_esp32c2 - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-component_ut_esp32h2 - - <<: *if-label-component_ut_esp32s2 - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32 - - <<: *if-label-custom_test_esp32c2 - - <<: *if-label-custom_test_esp32c3 - - <<: *if-label-custom_test_esp32h2 - - <<: *if-label-custom_test_esp32s2 - - <<: *if-label-custom_test_esp32s3 - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-example_test_esp32c2 - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-example_test_esp32h2 - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-example_test_esp32s3 - - <<: *if-label-host_test - - <<: *if-label-integration_test - - <<: *if-label-iperf_stress_test - - <<: *if-label-unit_test - - <<: *if-label-unit_test_esp32 - - <<: *if-label-unit_test_esp32c2 - - <<: *if-label-unit_test_esp32c3 - - <<: *if-label-unit_test_esp32h2 - - <<: *if-label-unit_test_esp32s2 - - <<: *if-label-unit_test_esp32s3 - - <<: *if-label-weekend_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-host_test - - <<: *if-dev-push - changes: *patterns-integration_test - - <<: *if-dev-push - changes: *patterns-unit_test - .rules:test:component_ut-esp32: rules: - <<: *if-revert-branch diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 5955974564..5f2f72b11a 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -173,8 +173,6 @@ component_ut_pytest_esp32c3_generic: .target_test_job_template: stage: target_test - needs: - - assign_test artifacts: when: always paths: @@ -204,6 +202,8 @@ component_ut_pytest_esp32c3_generic: .example_test_template: extends: .target_test_job_template + needs: + - assign_example_test variables: TEST_CASE_PATH: "$CI_PROJECT_DIR/examples" CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs" @@ -467,6 +467,8 @@ example_test_ESP32C3_SDSPI: .test_app_template: extends: .target_test_job_template + needs: + - assign_custom_test variables: TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps" CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs" @@ -558,9 +560,11 @@ test_app_test_flash_psram_f8r8: .unit_test_template: extends: .target_test_job_template + needs: # the assign already needs all the build jobs + - assign_unit_test variables: TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/unit-test-app" - CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/CIConfigs" + CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/test_configs" .unit_test_esp32_template: extends: @@ -867,14 +871,14 @@ UT_S3_FLASH: # needn't install idf python env - .before_script_minimal needs: - - assign_test + - assign_integration_test - build_ssc_esp32 variables: LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF" LOG_PATH: "${CI_PROJECT_DIR}/TEST_LOGS" TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/auto_test_script/TestCaseFiles" MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml" - CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs" + CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/test_configs" KNOWN_ISSUE_FILE: "${CI_PROJECT_DIR}/components/idf_test/integration_test/KnownIssues" CI_RUNNER_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIRunner.py" PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages diff --git a/components/idf_test/integration_test/CIConfigs/nvs_compatible_test_.yml b/components/idf_test/integration_test/test_configs/nvs_compatible_test_.yml similarity index 100% rename from components/idf_test/integration_test/CIConfigs/nvs_compatible_test_.yml rename to components/idf_test/integration_test/test_configs/nvs_compatible_test_.yml