From e6ad766d68598b3190c930a9cd67036aeb1ae3d9 Mon Sep 17 00:00:00 2001 From: Marius Vikhammer Date: Tue, 16 Jun 2020 19:00:27 +0800 Subject: [PATCH] CI: add script for checking links in example READMEs Closes IDF-1846 --- tools/ci/check_readme_links.py | 161 +++++++++++++++++++++++++++++++++ tools/ci/config/host-test.yml | 1 + tools/ci/config/pre_check.yml | 9 ++ tools/ci/executable-list.txt | 1 + 4 files changed, 172 insertions(+) create mode 100755 tools/ci/check_readme_links.py diff --git a/tools/ci/check_readme_links.py b/tools/ci/check_readme_links.py new file mode 100755 index 0000000000..f500eaac4f --- /dev/null +++ b/tools/ci/check_readme_links.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python +# +# Checks that all links in the readme markdown files are valid +# +# Copyright 2020 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import re +import os.path +import urllib.request +import urllib.error +import concurrent.futures +import argparse +from pathlib import Path +from collections import namedtuple, defaultdict + +EXCLUDE_DOCS_LIST = ['examples/peripherals/secure_element/atecc608_ecdsa/components/esp-cryptoauthlib/cryptoauthlib/**'] + +# The apple apps links are not accessible from the company network for some reason +EXCLUDE_URL_LIST = ['https://apps.apple.com/in/app/esp-ble-provisioning/id1473590141', 'https://apps.apple.com/in/app/esp-softap-provisioning/id1474040630'] + +Link = namedtuple('Link', ['file', 'url']) + + +class ReadmeLinkError(Exception): + def __init__(self, file, url): + self.file = file + self.url = url + + +class RelativeLinkError(ReadmeLinkError): + def __str__(self): + return "Relative link error, file - {} not found, linked from {}".format(self.url, self.file) + + +class UrlLinkError(ReadmeLinkError): + def __init__(self, file, url, error_code): + self.error_code = error_code + super().__init__(file, url) + + def __str__(self): + files = [str(f) for f in self.file] + return "URL error, url - {} in files - {} is not accessible, request returned {}".format(self.url, ", ".join(files), self.error_code) + + +# we do not want a failed test just due to bad network conditions, for non 404 errors we simply print a warning +def check_url(url, files, timeout): + try: + with urllib.request.urlopen(url, timeout=timeout): + return + except urllib.error.HTTPError as e: + if e.code == 404: + raise UrlLinkError(files, url, str(e)) + else: + print("Unable to access {}, err = {}".format(url, str(e))) + except Exception as e: + print("Unable to access {}, err = {}".format(url, str(e))) + + +def check_web_links(web_links): + + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + errors = [] + future_to_url = {executor.submit(check_url, url, files, timeout=30): (url, files) for url, files in web_links.items()} + for future in concurrent.futures.as_completed(future_to_url): + try: + future.result() + except UrlLinkError as e: + errors.append(e) + + return errors + + +def check_file_links(file_links): + errors = [] + + for link in file_links: + link_path = link.file.parent / link.url + + if not Path.exists(link_path): + errors.append(RelativeLinkError(link.file, link.url)) + + print("Found {} errors with relative links".format(len(errors))) + return errors + + +def get_md_links(folder): + MD_LINK_RE = r"\[.+?\]\((.+?)(#.+)?\)" + + idf_path = Path(os.getenv('IDF_PATH')) + links = [] + + for path in (idf_path / folder).rglob('*.md'): + if any([path.relative_to(idf_path).match(exclude_doc) for exclude_doc in EXCLUDE_DOCS_LIST]): + print("{} - excluded".format(path)) + continue + + with path.open(encoding='utf8') as f: + content = f.read() + + for url in re.findall(MD_LINK_RE, content): + link = Link(path, url[0].lstrip()) + # Ignore "local" links + if not link.url.startswith('#'): + links.append(link) + + return links + + +def check_readme_links(args): + + links = get_md_links('examples') + print("Found {} links".format(len(links))) + + errors = [] + + web_links = defaultdict(list) + file_links = [] + + # Sort links into file and web links + for link in links: + if link.url.startswith('http'): + web_links[link.url].append(link.file) + else: + file_links.append(link) + + for url in EXCLUDE_URL_LIST: + del web_links[url] + + errors.extend(check_file_links(file_links)) + + if not args.skip_weburl: + errors.extend(check_web_links(web_links)) + + print("Found {} errors:".format(len(errors))) + for e in errors: + print(e) + if errors: + raise e + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='check_readme_links.py: Checks for dead links in example READMEs', prog='check_readme_links.py') + parser.add_argument("--skip-weburl", "-w", action='store_true', help="Skip checking of web URLs, only check links to local files") + args = parser.parse_args() + + check_readme_links(args) diff --git a/tools/ci/config/host-test.yml b/tools/ci/config/host-test.yml index e7018ddc1b..7cff3fc6cb 100644 --- a/tools/ci/config/host-test.yml +++ b/tools/ci/config/host-test.yml @@ -312,3 +312,4 @@ test_mkdfu: script: - cd ${IDF_PATH}/tools/test_mkdfu - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_mkdfu.py + diff --git a/tools/ci/config/pre_check.yml b/tools/ci/config/pre_check.yml index d986503d82..89f50ed748 100644 --- a/tools/ci/config/pre_check.yml +++ b/tools/ci/config/pre_check.yml @@ -221,3 +221,12 @@ scan_tests: - python $CI_SCAN_TESTS_PY example_test -b make $EXAMPLE_TEST_DIR --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR - python $CI_SCAN_TESTS_PY example_test -b cmake $EXAMPLE_TEST_DIR --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR - python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR + +check_readme_links: + extends: .check_job_template + tags: [ "amd64", "deploy", "internet" ] + allow_failure: true + variables: + PYTHON_VER: 3 + script: + - python ${IDF_PATH}/tools/ci/check_readme_links.py diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 6834f29bcd..094a872214 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -44,6 +44,7 @@ tools/ci/check_deprecated_kconfigs.py tools/ci/check_examples_cmake_make.sh tools/ci/check_examples_rom_header.sh tools/ci/check_idf_version.sh +tools/ci/check_readme_links.py tools/ci/check_rom_apis.sh tools/ci/check_ut_cmake_make.sh tools/ci/checkout_project_ref.py