diff --git a/tools/check_python_dependencies.py b/tools/check_python_dependencies.py index c918d4d787..145a3a5d79 100755 --- a/tools/check_python_dependencies.py +++ b/tools/check_python_dependencies.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2018-2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse @@ -9,13 +9,22 @@ import re import sys try: - import pkg_resources + from packaging.requirements import Requirement + from packaging.version import Version except ImportError: - print('pkg_resources cannot be imported. The most common cause is a missing pip or setuptools package. ' - 'If you\'ve installed a custom Python then these packages are provided separately and have to be installed as well. ' + print('packaging cannot be imported. ' + 'If you\'ve installed a custom Python then this package is provided separately and have to be installed as well. ' 'Please refer to the Get Started section of the ESP-IDF Programming Guide for setting up the required packages.') sys.exit(1) +try: + from importlib.metadata import PackageNotFoundError, requires + from importlib.metadata import version as get_version +except ImportError: + # compatibility for python <=3.7 + from importlib_metadata import PackageNotFoundError, requires # type: ignore + from importlib_metadata import version as get_version # type: ignore + try: from typing import Set except ImportError: @@ -59,18 +68,31 @@ if __name__ == '__main__': if not name_m: print('Malformed input. Cannot find name in {}'.format(con)) sys.exit(1) - constr_dict[name_m[0]] = con + constr_dict[name_m[0]] = con.partition(' #')[0] # remove comments not_satisfied = [] # in string form which will be printed # already_checked set is used in order to avoid circular checks which would cause looping. - already_checked = set() # type: Set[pkg_resources.Requirement] + already_checked = set() # type: Set[Requirement] # required_set contains package names in string form without version constraints. If the package has a constraint # specification (package name + version requirement) then use that instead. new_req_list is used to store # requirements to be checked on each level of breath-first-search of the package dependency tree. The initial # version is the direct dependencies deduced from the requirements arguments of the script. - new_req_list = [pkg_resources.Requirement.parse(constr_dict.get(i, i)) for i in required_set] + new_req_list = [Requirement(constr_dict.get(i, i)) for i in required_set] + + def version_check(requirement: Requirement) -> None: + # compare installed version with required + version = Version(get_version(requirement.name)) + if version.base_version not in requirement.specifier: + not_satisfied.append(f"Requirement '{requirement}' was not met. Installed version: {version}") + + # evaluate markers and check versions of direct requirements + for req in new_req_list[:]: + if not req.marker or req.marker.evaluate(): + version_check(req) + else: + new_req_list.remove(req) while new_req_list: req_list = new_req_list @@ -78,20 +100,23 @@ if __name__ == '__main__': already_checked.update(req_list) for requirement in req_list: # check one level of the dependency tree try: - dependency_requirements = set(pkg_resources.get_distribution(requirement).requires()) + dependency_requirements = set() + extras = list(requirement.extras) or [''] + for name in requires(requirement.name) or []: + sub_req = Requirement(name) + # check extras e.g. esptool[hsm] + for extra in extras: + # evaluate markers if present + if not sub_req.marker or sub_req.marker.evaluate(environment={'extra': extra}): + dependency_requirements.add(sub_req) + version_check(sub_req) # dependency_requirements are the direct dependencies of "requirement". They belong to the next level # of the dependency tree. They will be checked only if they haven't been already. Note that the # version is taken into account as well because packages can have different requirements for a given # Python package. The dependencies need to be checked for all of them because they can be different. new_req_list.extend(dependency_requirements - already_checked) - except pkg_resources.ResolutionError as e: - not_satisfied.append(' - '.join([str(requirement), str(e)])) - except IndexError: - # If the requirement is not installed because of a marker (requirement.marker), for example different - # operating system or python version, then pkg_resources.get_distribution() will fail with IndexError. - # We could avoid this by checking packaging.markers.Marker(requirement.marker).evaluate() but it would - # add dependency on packaging. - pass + except PackageNotFoundError as e: + not_satisfied.append(f"'{e}' - was not found and is required by the application") if len(not_satisfied) > 0: print('The following Python requirements are not satisfied:') diff --git a/tools/ci/build_template_app.sh b/tools/ci/build_template_app.sh index 140aad2c73..b1f6c7b9f9 100755 --- a/tools/ci/build_template_app.sh +++ b/tools/ci/build_template_app.sh @@ -62,8 +62,7 @@ build_stage2() { --build-log ${BUILD_LOG_CMAKE} \ --size-file size.json \ --collect-size-info size_info.txt \ - --default-build-targets esp32,esp32s2,esp32s3,esp32c2,esp32c3 \ - --ignore-warning-str "DeprecationWarning: pkg_resources is deprecated as an API" + --default-build-targets esp32,esp32s2,esp32s3,esp32c2,esp32c3 # add esp32h2 back after IDF-5541 } @@ -78,8 +77,7 @@ build_stage1() { --build-log ${BUILD_LOG_CMAKE} \ --size-file size.json \ --collect-size-info size_info.txt \ - --default-build-targets esp32,esp32s2,esp32s3,esp32c2,esp32c3,esp32h2 \ - --ignore-warning-str "DeprecationWarning: pkg_resources is deprecated as an API" + --default-build-targets esp32,esp32s2,esp32s3,esp32c2,esp32c3,esp32h2 } # Default arguments diff --git a/tools/ci/ignore_build_warnings.txt b/tools/ci/ignore_build_warnings.txt index a52d0010d2..0f2daf214b 100644 --- a/tools/ci/ignore_build_warnings.txt +++ b/tools/ci/ignore_build_warnings.txt @@ -15,4 +15,3 @@ CryptographyDeprecationWarning Warning: \d+/\d+ app partitions are too small for binary CMake Deprecation Warning at main/lib/tinyxml2/CMakeLists\.txt:11 \(cmake_policy\) The smallest .+ partition is nearly full \(\d+% free space left\)! -DeprecationWarning: pkg_resources is deprecated as an API diff --git a/tools/requirements/requirements.core.txt b/tools/requirements/requirements.core.txt index fc3cb0b48a..8d4984c82a 100644 --- a/tools/requirements/requirements.core.txt +++ b/tools/requirements/requirements.core.txt @@ -1,6 +1,9 @@ # Python package requirements for ESP-IDF. These are the so called core features which are installed in all systems. setuptools +packaging +# importlib_metadata: is part of python3.8 and newer as importlib.metadata +importlib_metadata; python_version < "3.8" click pyserial future