mirror of
https://github.com/espressif/esp-idf
synced 2025-03-09 17:19:09 -04:00
8d248d8903
The check_public_headers.py script does not function well with MINIMAL_BUILD enabled, as some headers from excluded components may be missing. While it might be possible to address this issue with exclusions, it is likely more effective and provides more coverage to include all components, not just those required by the blink example. Therefore, MINIMAL_BUILD has been disabled for this test, ensuring that all registered components are included in the build. If defined, the COMPONENTS variable takes precedence over the MINIMAL_BUILD property, so setting "-DCOMPONENTS=" overrides the MINIMAL_BUILD property. Signed-off-by: Frantisek Hrbata <frantisek.hrbata@espressif.com>
410 lines
21 KiB
Python
410 lines
21 KiB
Python
#!/usr/bin/env python
|
|
#
|
|
# Checks all public headers in IDF in the ci
|
|
#
|
|
# SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
import argparse
|
|
import fnmatch
|
|
import json
|
|
import os
|
|
import queue
|
|
import re
|
|
import subprocess
|
|
import tempfile
|
|
from threading import Event
|
|
from threading import Thread
|
|
from typing import List
|
|
from typing import Optional
|
|
from typing import Set
|
|
from typing import Tuple
|
|
from typing import Union
|
|
|
|
|
|
class HeaderFailed(Exception):
|
|
"""Base header failure exception"""
|
|
pass
|
|
|
|
|
|
class HeaderFailedSdkconfig(HeaderFailed):
|
|
def __str__(self) -> str:
|
|
return 'Sdkconfig Error'
|
|
|
|
|
|
class HeaderFailedBuildError(HeaderFailed):
|
|
def __init__(self, compiler: str):
|
|
self.compiler = compiler
|
|
|
|
def __str__(self) -> str:
|
|
return 'Header Build Error with {}'.format(self.compiler)
|
|
|
|
|
|
class HeaderFailedPreprocessError(HeaderFailed):
|
|
def __str__(self) -> str:
|
|
return 'Header Procecessing Error'
|
|
|
|
|
|
class HeaderFailedCppGuardMissing(HeaderFailed):
|
|
def __str__(self) -> str:
|
|
return 'Header Missing C++ Guard'
|
|
|
|
|
|
class HeaderFailedContainsCode(HeaderFailed):
|
|
def __str__(self) -> str:
|
|
return 'Header Produced non-zero object'
|
|
|
|
|
|
class HeaderFailedContainsStaticAssert(HeaderFailed):
|
|
def __str__(self) -> str:
|
|
return 'Header uses _Static_assert or static_assert instead of ESP_STATIC_ASSERT'
|
|
|
|
|
|
# Creates a temp file and returns both output as a string and a file name
|
|
#
|
|
def exec_cmd_to_temp_file(what: List, suffix: str='') -> Tuple[int, str, str, str, str]:
|
|
out_file = tempfile.NamedTemporaryFile(suffix=suffix, delete=False)
|
|
rc, out, err, cmd = exec_cmd(what, out_file)
|
|
with open(out_file.name, 'r', encoding='utf-8') as f:
|
|
out = f.read()
|
|
return rc, out, err, out_file.name, cmd
|
|
|
|
|
|
def exec_cmd(what: List, out_file: Union['tempfile._TemporaryFileWrapper[bytes]', int]=subprocess.PIPE) -> Tuple[int, str, str, str]:
|
|
p = subprocess.Popen(what, stdin=subprocess.PIPE, stdout=out_file, stderr=subprocess.PIPE)
|
|
output_b, err_b = p.communicate()
|
|
rc = p.returncode
|
|
output: str = output_b.decode('utf-8') if output_b is not None else ''
|
|
err: str = err_b.decode('utf-8') if err_b is not None else ''
|
|
return rc, output, err, ' '.join(what)
|
|
|
|
|
|
class PublicHeaderChecker:
|
|
|
|
def log(self, message: str, debug: bool=False) -> None:
|
|
if self.verbose or debug:
|
|
print(message)
|
|
|
|
def __init__(self, verbose: bool=False, jobs: int=1, prefix: Optional[str]=None) -> None:
|
|
self.gcc = '{}gcc'.format(prefix)
|
|
self.gpp = '{}g++'.format(prefix)
|
|
self.verbose = verbose
|
|
self.jobs = jobs
|
|
self.prefix = prefix
|
|
self.extern_c = re.compile(r'extern "C"')
|
|
self.error_macro = re.compile(r'#error')
|
|
self.error_orphan_kconfig = re.compile(r'#error CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED')
|
|
self.kconfig_macro = re.compile(r'\bCONFIG_[A-Z0-9_]+')
|
|
self.static_assert = re.compile(r'(_Static_assert|static_assert)')
|
|
self.defines_assert = re.compile(r'#define[ \t]+ESP_STATIC_ASSERT')
|
|
self.auto_soc_header = re.compile(r'components/soc/esp[a-z0-9_]+(?:/\w+)?/(include|register)/(soc|modem)/[a-zA-Z0-9_]+.h')
|
|
self.assembly_nocode = r'^\s*(\.file|\.text|\.ident|\.option|\.attribute|(\.section)?).*$'
|
|
self.check_threads: List[Thread] = []
|
|
self.stdc = '--std=c99'
|
|
self.stdcpp = '--std=c++17'
|
|
|
|
self.job_queue: queue.Queue = queue.Queue()
|
|
self.failed_queue: queue.Queue = queue.Queue()
|
|
self.terminate = Event()
|
|
|
|
def __enter__(self) -> 'PublicHeaderChecker':
|
|
for i in range(self.jobs):
|
|
t = Thread(target=self.check_headers, args=(i, ))
|
|
self.check_threads.append(t)
|
|
t.start()
|
|
return self
|
|
|
|
def __exit__(self, exc_type: str, exc_value: str, traceback: str) -> None:
|
|
self.terminate.set()
|
|
for t in self.check_threads:
|
|
t.join()
|
|
|
|
# thread function process incoming header file from a queue
|
|
def check_headers(self, num: int) -> None:
|
|
while not self.terminate.is_set():
|
|
if not self.job_queue.empty():
|
|
task = self.job_queue.get()
|
|
if task is None:
|
|
self.terminate.set()
|
|
else:
|
|
try:
|
|
self.check_one_header(task, num)
|
|
except HeaderFailed as e:
|
|
self.failed_queue.put('{}: Failed! {}'.format(task, e))
|
|
except Exception as e:
|
|
# Makes sure any unexpected exceptions causes the program to terminate
|
|
self.failed_queue.put('{}: Failed! {}'.format(task, e))
|
|
self.terminate.set()
|
|
raise
|
|
|
|
def get_failed(self) -> List:
|
|
return list(self.failed_queue.queue)
|
|
|
|
def join(self) -> None:
|
|
for t in self.check_threads:
|
|
while t.is_alive() and not self.terminate.is_set():
|
|
t.join(1) # joins with timeout to respond to keyboard interrupt
|
|
|
|
# Checks one header procedure:
|
|
# - Preprocess the include file with C preprocessor and with CPP preprocessor, compare check for possible issues
|
|
# - Compile the header with both C and C++ compiler
|
|
def check_one_header(self, header: str, num: int) -> None:
|
|
self.preprocess_one_header(header, num)
|
|
self.compile_one_header_with(self.gcc, self.stdc, header)
|
|
self.compile_one_header_with(self.gpp, self.stdcpp, header)
|
|
|
|
# Checks if the header contains some assembly code and whether it is compilable
|
|
def compile_one_header_with(self, compiler: str, std_flags: str, header: str) -> None:
|
|
rc, out, err, cmd = exec_cmd([compiler, std_flags, '-S', '-o-', '-include', header, self.main_c] + self.include_dir_flags)
|
|
if rc == 0:
|
|
if not re.sub(self.assembly_nocode, '', out, flags=re.M).isspace():
|
|
raise HeaderFailedContainsCode()
|
|
return # Header OK: produced zero code
|
|
self.log('{}: FAILED: compilation issue'.format(header), True)
|
|
self.log(err, True)
|
|
self.log('\nCompilation command failed:\n{}\n'.format(cmd), True)
|
|
raise HeaderFailedBuildError(compiler)
|
|
|
|
# Checks one header using preprocessing and parsing
|
|
# 1) Remove comments and check
|
|
# - if we have some `CONFIG_...` macros (for test 2)
|
|
# - static asserts
|
|
# 2) Preprocess with C++ flags and test orphan Kconfig macros, or compiler error
|
|
# 3) Preprocess with C flags and test for compiler errors
|
|
# 4) Compare outputs from steps 2) and 3)
|
|
# - outputs are the same, but contain only whitespaces -> header is OK (contains only preprocessor directives)
|
|
# - outputs are the same, but contain some code -> FAIL the test, we're missing `extern "C"` somewhere
|
|
# - outputs are different:
|
|
# - check for `extern "C"` in the diff
|
|
# - Not present? -> FAIL the test, we're missing `extern "C"` somewhere
|
|
# - Present? -- extern "C" is there, but could be from included headers
|
|
# - check for `extern "C"` in the orig header (output of step 1)
|
|
# - Present? -> header is OK (we're have the `extern "C"` in the header under test)
|
|
# - Not present? -- we're missing `extern "C"` in our header, but do we really need it?
|
|
# - Remove all directives and harmless macro invocations from our current header
|
|
# - We still have some code? -> FAIL the test (our header needs extern "C")
|
|
# - Only whitespaces -> header is OK (it contains only macros and directives)
|
|
def preprocess_one_header(self, header: str, num: int) -> None:
|
|
all_compilation_flags = ['-w', '-P', '-E', '-DESP_PLATFORM', '-include', header, self.main_c] + self.include_dir_flags
|
|
# just strip comments to check for CONFIG_... macros or static asserts
|
|
rc, out, err, _ = exec_cmd([self.gcc, '-fpreprocessed', '-dD', '-P', '-E', header] + self.include_dir_flags)
|
|
# we ignore the rc here, as the `-fpreprocessed` flag expects the file to have macros already expanded, so we might get some errors
|
|
# here we use it only to remove comments (even if the command returns non-zero code it produces the correct output)
|
|
if re.search(self.kconfig_macro, out):
|
|
# enable defined #error if sdkconfig.h not included
|
|
all_compilation_flags.append('-DIDF_CHECK_SDKCONFIG_INCLUDED')
|
|
# If the file contain _Static_assert or static_assert, make sure it doesn't not define ESP_STATIC_ASSERT and that it
|
|
# is not an automatically generated soc header file
|
|
grp = re.search(self.static_assert, out)
|
|
# Normalize the potential A//B, A/./B, A/../A, from the name
|
|
normalized_path = os.path.normpath(header)
|
|
if grp and not re.search(self.defines_assert, out) and not re.search(self.auto_soc_header, normalized_path):
|
|
self.log('{}: FAILED: contains {}. Please use ESP_STATIC_ASSERT'.format(header, grp.group(1)), True)
|
|
raise HeaderFailedContainsStaticAssert()
|
|
try:
|
|
# compile with C++, check for errors, outputs for a temp file
|
|
rc, cpp_out, err, cpp_out_file, cmd = exec_cmd_to_temp_file([self.gpp, self.stdcpp] + all_compilation_flags)
|
|
if rc != 0:
|
|
if re.search(self.error_macro, err):
|
|
if re.search(self.error_orphan_kconfig, err):
|
|
self.log('{}: CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED'.format(header), True)
|
|
raise HeaderFailedSdkconfig()
|
|
self.log('{}: Error directive failure: OK'.format(header))
|
|
return
|
|
self.log('{}: FAILED: compilation issue'.format(header), True)
|
|
self.log(err, True)
|
|
self.log('\nCompilation command failed:\n{}\n'.format(cmd), True)
|
|
raise HeaderFailedPreprocessError()
|
|
# compile with C compiler, outputs to another temp file
|
|
rc, _, err, c_out_file, _ = exec_cmd_to_temp_file([self.gcc, self.stdc] + all_compilation_flags)
|
|
if rc != 0:
|
|
self.log('{} FAILED: compilation in C (while C++ compilation passes)'.format(header))
|
|
raise HeaderFailedPreprocessError()
|
|
# diff the two outputs
|
|
rc, diff, err, _ = exec_cmd(['diff', c_out_file, cpp_out_file])
|
|
if not diff or diff.isspace():
|
|
if not cpp_out or cpp_out.isspace():
|
|
self.log('{} The same, but empty out - OK'.format(header))
|
|
return
|
|
self.log('{} FAILED C and C++ preprocessor output is the same!'.format(header), True)
|
|
raise HeaderFailedCppGuardMissing()
|
|
if re.search(self.extern_c, diff):
|
|
self.log('{} extern C present in the diff'.format(header))
|
|
# now check the extern "C" is really in the unprocessed header
|
|
if re.search(self.extern_c, out):
|
|
self.log('{} extern C present in the actual header, too - OK'.format(header))
|
|
return
|
|
# at this point we know that the header itself is missing extern-C, so we need to check if it contains an actual *code*
|
|
# we remove all preprocessor's directive to check if there's any code besides macros
|
|
macros = re.compile(r'(?m)^\s*#(?:.*\\\r?\n)*.*$') # Matches multiline preprocessor directives
|
|
without_macros = macros.sub('', out)
|
|
if without_macros.isspace():
|
|
self.log("{} Header doesn't need extern-C, it's all just macros - OK".format(header))
|
|
return
|
|
# at this point we know that the header is not only composed of macro definitions, but could just contain some "harmless" macro calls
|
|
# let's remove them and check again
|
|
macros_calls = r'(.*?)ESP_STATIC_ASSERT[^;]+;' # static assert macro only, we could add more if needed
|
|
without_macros = re.sub(macros_calls, '', without_macros, flags=re.DOTALL)
|
|
if without_macros.isspace():
|
|
self.log("{} Header doesn't need extern-C, it's all macros definitions and calls - OK".format(header))
|
|
return
|
|
|
|
self.log('{} Different but no extern C - FAILED'.format(header), True)
|
|
raise HeaderFailedCppGuardMissing()
|
|
finally:
|
|
os.unlink(cpp_out_file)
|
|
try:
|
|
os.unlink(c_out_file)
|
|
except Exception:
|
|
pass
|
|
|
|
# Get compilation data from an example to list all public header files
|
|
def list_public_headers(self, ignore_dirs: List, ignore_files: Union[List, Set], only_dir: Optional[str]=None) -> None:
|
|
idf_path = os.getenv('IDF_PATH')
|
|
if idf_path is None:
|
|
raise RuntimeError("Environment variable 'IDF_PATH' wasn't set.")
|
|
project_dir = os.path.join(idf_path, 'examples', 'get-started', 'blink')
|
|
build_dir = tempfile.mkdtemp()
|
|
sdkconfig = os.path.join(build_dir, 'sdkconfig')
|
|
try:
|
|
os.unlink(os.path.join(project_dir, 'sdkconfig'))
|
|
except FileNotFoundError:
|
|
pass
|
|
subprocess.check_call(['idf.py', '-B', build_dir, f'-DSDKCONFIG={sdkconfig}', '-DCOMPONENTS=', 'reconfigure'],
|
|
cwd=project_dir)
|
|
|
|
def get_std(json: List, extension: str) -> str:
|
|
# compile commands for the files with specified extension, containing C(XX) standard flag
|
|
command = [c for c in j if c['file'].endswith('.' + extension) and '-std=' in c['command']][0]
|
|
return str([s for s in command['command'].split() if 'std=' in s][0]) # grab the std flag
|
|
|
|
build_commands_json = os.path.join(build_dir, 'compile_commands.json')
|
|
with open(build_commands_json, 'r', encoding='utf-8') as f:
|
|
j = json.load(f)
|
|
self.stdc = get_std(j, 'c')
|
|
self.stdcpp = get_std(j, 'cpp')
|
|
build_command = j[0]['command'].split()
|
|
include_dir_flags = []
|
|
include_dirs = []
|
|
# process compilation flags (includes and defines)
|
|
for item in build_command:
|
|
if item.startswith('-I'):
|
|
include_dir_flags.append(item)
|
|
if 'components' in item:
|
|
include_dirs.append(item[2:]) # Removing the leading "-I"
|
|
if item.startswith('-D'):
|
|
include_dir_flags.append(item.replace('\\','')) # removes escaped quotes, eg: -DMBEDTLS_CONFIG_FILE=\\\"mbedtls/esp_config.h\\\"
|
|
include_dir_flags.append('-I' + os.path.join(build_dir, 'config'))
|
|
include_dir_flags.append('-DCI_HEADER_CHECK')
|
|
sdkconfig_h = os.path.join(build_dir, 'config', 'sdkconfig.h')
|
|
# prepares a main_c file for easier sdkconfig checks and avoid compilers warning when compiling headers directly
|
|
with open(sdkconfig_h, 'a') as f:
|
|
f.write('#define IDF_SDKCONFIG_INCLUDED')
|
|
main_c = os.path.join(build_dir, 'compile.c')
|
|
with open(main_c, 'w') as f:
|
|
f.write('#if defined(IDF_CHECK_SDKCONFIG_INCLUDED) && ! defined(IDF_SDKCONFIG_INCLUDED)\n'
|
|
'#error CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED\n'
|
|
'#endif')
|
|
# processes public include dirs, removing ignored files
|
|
all_include_files = []
|
|
files_to_check = []
|
|
for d in include_dirs:
|
|
if only_dir is not None and not os.path.relpath(d, idf_path).startswith(os.path.relpath(only_dir, idf_path)):
|
|
self.log('{} - directory ignored (not in "{}")'.format(d, only_dir))
|
|
continue
|
|
if os.path.relpath(d, idf_path).startswith(tuple(ignore_dirs)):
|
|
self.log('{} - directory ignored'.format(d))
|
|
continue
|
|
for root, dirnames, filenames in os.walk(d):
|
|
for filename in fnmatch.filter(filenames, '*.h'):
|
|
all_include_files.append(os.path.join(root, filename))
|
|
self.main_c = main_c
|
|
self.include_dir_flags = include_dir_flags
|
|
ignore_files = set(ignore_files)
|
|
# processes public include files, removing ignored files
|
|
for file_name in all_include_files:
|
|
rel_path_file = os.path.relpath(file_name, idf_path)
|
|
if any([os.path.commonprefix([d, rel_path_file]) == d for d in ignore_dirs]):
|
|
self.log('{} - file ignored (inside ignore dir)'.format(file_name))
|
|
continue
|
|
if rel_path_file in ignore_files:
|
|
self.log('{} - file ignored'.format(file_name))
|
|
continue
|
|
files_to_check.append(file_name)
|
|
# removes duplicates and places headers to a work queue
|
|
for file_name in set(files_to_check):
|
|
self.job_queue.put(file_name)
|
|
self.job_queue.put(None) # to indicate the last job
|
|
|
|
|
|
def check_all_headers() -> None:
|
|
parser = argparse.ArgumentParser('Public header checker file', formatter_class=argparse.RawDescriptionHelpFormatter, epilog='''\
|
|
Tips for fixing failures reported by this script
|
|
------------------------------------------------
|
|
This checker validates all public headers to detect these types of issues:
|
|
1) "Sdkconfig Error": Using SDK config macros without including "sdkconfig.h"
|
|
* Check if the failing include file or any other included file uses "CONFIG_..." prefixed macros
|
|
2) "Header Build Error": Header itself is not compilable (missing includes, macros, types)
|
|
* Check that all referenced macros, types are available (defined or included)
|
|
* Check that all included header files are available (included in paths)
|
|
* Check for possible compilation issues
|
|
* If only the C++ compilation fails, check that the header is C++ compatible
|
|
* Try to compile only the offending header file
|
|
3) "Header Missing C++ Guard": Preprocessing the header by C and C++ should produce different output
|
|
* Check if the "#ifdef __cplusplus" header sentinels are present
|
|
4) "Header Produced non-zero object": Header contains some object, a definition
|
|
* Check if no definition is present in the offending header file
|
|
5) "Header contains _Static_assert or static_assert": Makes the use of _Static_assert or static_assert
|
|
functions instead of using ESP_STATIC_ASSERT macro
|
|
|
|
Notes:
|
|
* The script validates *all* header files (recursively) in public folders for all components.
|
|
* The script locates include paths from running a default build of "examples/get-started/blink'
|
|
* The script does not support any other targets than esp32
|
|
|
|
General tips:
|
|
* Use "-d" argument to make the script check only the offending header file
|
|
* Use "-v" argument to produce more verbose output
|
|
* Copy, paste and execute the compilation commands to reproduce build errors (script prints out
|
|
the entire compilation command line with absolute paths)
|
|
''')
|
|
parser.add_argument('--verbose', '-v', help='enables verbose mode', action='store_true')
|
|
parser.add_argument('--jobs', '-j', help='number of jobs to run checker', default=1, type=int)
|
|
parser.add_argument('--prefix', '-p', help='compiler prefix', default='xtensa-esp32-elf-', type=str)
|
|
parser.add_argument('--exclude-file', '-e', help='exception file', default='check_public_headers_exceptions.txt', type=str)
|
|
parser.add_argument('--only-dir', '-d', help='reduce the analysis to this directory only', default=None, type=str)
|
|
args = parser.parse_args()
|
|
|
|
# process excluded files and dirs
|
|
exclude_file = os.path.join(os.path.dirname(__file__), args.exclude_file)
|
|
with open(exclude_file, 'r', encoding='utf-8') as f:
|
|
lines = [line.rstrip() for line in f]
|
|
ignore_files = []
|
|
ignore_dirs = []
|
|
for line in lines:
|
|
if not line or line.isspace() or line.startswith('#'):
|
|
continue
|
|
if os.path.isdir(line):
|
|
ignore_dirs.append(line)
|
|
else:
|
|
ignore_files.append(line)
|
|
|
|
# start header check
|
|
with PublicHeaderChecker(args.verbose, args.jobs, args.prefix) as header_check:
|
|
header_check.list_public_headers(ignore_dirs, ignore_files, only_dir=args.only_dir)
|
|
try:
|
|
header_check.join()
|
|
failures = header_check.get_failed()
|
|
if len(failures) > 0:
|
|
for failed in failures:
|
|
print(failed)
|
|
print(parser.epilog)
|
|
exit(1)
|
|
print('No errors found')
|
|
except KeyboardInterrupt:
|
|
print('Keyboard interrupt')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
check_all_headers()
|