From 1418efe7a40b5e2a2c2210e6cc720180a6da2c25 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 17 Jun 2020 16:03:49 +0800 Subject: [PATCH 1/5] ci: retry download if catched IOError/EOFError --- tools/ci/python_packages/gitlab_api.py | 29 ++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index a5d58906d1..db86f0e893 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -4,6 +4,7 @@ import argparse import tempfile import tarfile import zipfile +from functools import wraps import gitlab @@ -11,6 +12,8 @@ import gitlab class Gitlab(object): JOB_NAME_PATTERN = re.compile(r"(\w+)(\s+(\d+)/(\d+))?") + DOWNLOAD_ERROR_MAX_RETRIES = 3 + def __init__(self, project_id=None): config_data_from_env = os.getenv("PYTHON_GITLAB_CONFIG") if config_data_from_env: @@ -69,6 +72,31 @@ class Gitlab(object): with zipfile.ZipFile(temp_file.name, "r") as archive_file: archive_file.extractall(destination) + def retry_download(func): + """ + This wrapper will only catch IOError and retry the whole function. + + So only use it with download functions, read() inside and atomic + functions + """ + @wraps(func) + def wrapper(self, *args, **kwargs): + retried = 0 + while True: + try: + res = func(self, *args, **kwargs) + except (IOError, EOFError) as e: + retried += 1 + if retried > self.DOWNLOAD_ERROR_MAX_RETRIES: + raise e # get out of the loop + else: + print('Retried for the {} time'.format(retried)) + continue + else: + break + return res + return wrapper + def download_artifact(self, job_id, artifact_path, destination=None): """ download specific path of job artifacts and extract to destination. @@ -123,6 +151,7 @@ class Gitlab(object): job_id_list.append({"id": job.id, "parallel_num": match.group(3)}) return job_id_list + @retry_download def download_archive(self, ref, destination, project_id=None): """ Download archive of certain commit of a repository and extract to destination path From 6b3f2ea744a38c18aa522143747ff64b50dbd3c3 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Wed, 16 Sep 2020 20:52:03 +0200 Subject: [PATCH 2/5] ci: fetch submodules from mirror, if enabled --- tools/ci/python_packages/gitlab_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index db86f0e893..e8aef55763 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -24,7 +24,8 @@ class Gitlab(object): else: # otherwise try to use config file at local filesystem config_files = None - self.gitlab_inst = gitlab.Gitlab.from_config(config_files=config_files) + gitlab_id = os.getenv("LOCAL_GITLAB_HTTPS_HOST") # if None, will use the default gitlab server + self.gitlab_inst = gitlab.Gitlab.from_config(gitlab_id=gitlab_id, config_files=config_files) self.gitlab_inst.auth() if project_id: self.project = self.gitlab_inst.projects.get(project_id) From 46ef92abee223d9330777711b221eb1fd9d58f7f Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 8 Jun 2021 15:04:06 +0800 Subject: [PATCH 3/5] ci: retry download artifacts --- tools/ci/python_packages/gitlab_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index e8aef55763..f2e24162a4 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -98,6 +98,7 @@ class Gitlab(object): return res return wrapper + @retry_download # type: ignore def download_artifact(self, job_id, artifact_path, destination=None): """ download specific path of job artifacts and extract to destination. @@ -152,7 +153,7 @@ class Gitlab(object): job_id_list.append({"id": job.id, "parallel_num": match.group(3)}) return job_id_list - @retry_download + @retry_download # type: ignore def download_archive(self, ref, destination, project_id=None): """ Download archive of certain commit of a repository and extract to destination path From 3b957bb67ea8615ffe651ca07aa1f2d511b58e53 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Thu, 14 Oct 2021 14:19:50 +0200 Subject: [PATCH 4/5] ci: retry Gitlab operations on error 500 --- tools/ci/python_packages/gitlab_api.py | 100 +++++++++++++++---------- 1 file changed, 59 insertions(+), 41 deletions(-) diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index f2e24162a4..8784e9e6e9 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -1,30 +1,66 @@ +import argparse import os import re -import argparse -import tempfile import tarfile +import tempfile +import time import zipfile from functools import wraps +from typing import Any, Callable, Dict, List, Optional import gitlab +TR = Callable[..., Any] + + +def retry(func: TR) -> TR: + """ + This wrapper will only catch several exception types associated with + "network issues" and retry the whole function. + """ + @wraps(func) + def wrapper(self: 'Gitlab', *args: Any, **kwargs: Any) -> Any: + retried = 0 + while True: + try: + res = func(self, *args, **kwargs) + except (IOError, EOFError, gitlab.exceptions.GitlabError) as e: + if isinstance(e, gitlab.exceptions.GitlabError) and e.response_code != 500: + # Only retry on error 500 + raise e + retried += 1 + if retried > self.DOWNLOAD_ERROR_MAX_RETRIES: + raise e # get out of the loop + else: + print('Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'), retried)) + time.sleep(2 ** retried) # wait a bit more after each retry + continue + else: + break + return res + return wrapper + class Gitlab(object): JOB_NAME_PATTERN = re.compile(r"(\w+)(\s+(\d+)/(\d+))?") DOWNLOAD_ERROR_MAX_RETRIES = 3 - def __init__(self, project_id=None): + def __init__(self, project_id: Optional[int] = None): config_data_from_env = os.getenv("PYTHON_GITLAB_CONFIG") if config_data_from_env: # prefer to load config from env variable - with tempfile.NamedTemporaryFile("w", delete=False) as temp_file: + with tempfile.NamedTemporaryFile('w', delete=False) as temp_file: temp_file.write(config_data_from_env) - config_files = [temp_file.name] + config_files = [temp_file.name] # type: Optional[List[str]] else: # otherwise try to use config file at local filesystem config_files = None - gitlab_id = os.getenv("LOCAL_GITLAB_HTTPS_HOST") # if None, will use the default gitlab server + self._init_gitlab_inst(project_id, config_files) + + @retry + def _init_gitlab_inst(self, project_id: Optional[int], config_files: Optional[List[str]]) -> None: + gitlab_id = os.getenv('LOCAL_GITLAB_HTTPS_HOST') # if None, will use the default gitlab server self.gitlab_inst = gitlab.Gitlab.from_config(gitlab_id=gitlab_id, config_files=config_files) self.gitlab_inst.auth() if project_id: @@ -32,7 +68,8 @@ class Gitlab(object): else: self.project = None - def get_project_id(self, name, namespace=None): + @retry + def get_project_id(self, name: str, namespace: Optional[str] = None) -> int: """ search project ID by name @@ -56,9 +93,10 @@ class Gitlab(object): if not res: raise ValueError("Can't find project") - return res[0] + return int(res[0]) - def download_artifacts(self, job_id, destination): + @retry + def download_artifacts(self, job_id: int, destination: str) -> None: """ download full job artifacts and extract to destination. @@ -73,33 +111,8 @@ class Gitlab(object): with zipfile.ZipFile(temp_file.name, "r") as archive_file: archive_file.extractall(destination) - def retry_download(func): - """ - This wrapper will only catch IOError and retry the whole function. - - So only use it with download functions, read() inside and atomic - functions - """ - @wraps(func) - def wrapper(self, *args, **kwargs): - retried = 0 - while True: - try: - res = func(self, *args, **kwargs) - except (IOError, EOFError) as e: - retried += 1 - if retried > self.DOWNLOAD_ERROR_MAX_RETRIES: - raise e # get out of the loop - else: - print('Retried for the {} time'.format(retried)) - continue - else: - break - return res - return wrapper - - @retry_download # type: ignore - def download_artifact(self, job_id, artifact_path, destination=None): + @retry + def download_artifact(self, job_id: int, artifact_path: str, destination: Optional[str] = None) -> List[bytes]: """ download specific path of job artifacts and extract to destination. @@ -114,7 +127,7 @@ class Gitlab(object): for a_path in artifact_path: try: - data = job.artifact(a_path) + data = job.artifact(a_path) # type: bytes except gitlab.GitlabGetError as e: print("Failed to download '{}' form job {}".format(a_path, job_id)) raise e @@ -131,7 +144,8 @@ class Gitlab(object): return raw_data_list - def find_job_id(self, job_name, pipeline_id=None, job_status="success"): + @retry + def find_job_id(self, job_name: str, pipeline_id: Optional[str] = None, job_status: str = 'success') -> List[Dict]: """ Get Job ID from job name of specific pipeline @@ -153,8 +167,8 @@ class Gitlab(object): job_id_list.append({"id": job.id, "parallel_num": match.group(3)}) return job_id_list - @retry_download # type: ignore - def download_archive(self, ref, destination, project_id=None): + @retry + def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None) -> str: """ Download archive of certain commit of a repository and extract to destination path @@ -184,7 +198,7 @@ class Gitlab(object): return os.path.join(os.path.realpath(destination), root_name) -if __name__ == '__main__': +def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("action") parser.add_argument("project_id", type=int) @@ -210,3 +224,7 @@ if __name__ == '__main__': elif args.action == "get_project_id": ret = gitlab_inst.get_project_id(args.project_name) print("project id: {}".format(ret)) + + +if __name__ == '__main__': + main() From 672d112395761df80f88c6b05761f20d2173fd68 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Fri, 15 Oct 2021 10:57:08 +0200 Subject: [PATCH 5/5] ci: revert type annotations in gitlab_api.py to python2 compatible --- tools/ci/python_packages/gitlab_api.py | 27 ++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index 8784e9e6e9..8e8df4b231 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -6,20 +6,23 @@ import tempfile import time import zipfile from functools import wraps -from typing import Any, Callable, Dict, List, Optional import gitlab -TR = Callable[..., Any] +try: + from typing import Any, Callable, Dict, List, Optional + TR = Callable[..., Any] +except ImportError: + pass -def retry(func: TR) -> TR: +def retry(func): # type: (TR) -> TR """ This wrapper will only catch several exception types associated with "network issues" and retry the whole function. """ @wraps(func) - def wrapper(self: 'Gitlab', *args: Any, **kwargs: Any) -> Any: + def wrapper(self, *args, **kwargs): # type: (Gitlab, Any, Any) -> Any retried = 0 while True: try: @@ -46,7 +49,7 @@ class Gitlab(object): DOWNLOAD_ERROR_MAX_RETRIES = 3 - def __init__(self, project_id: Optional[int] = None): + def __init__(self, project_id=None): # type: (Optional[int]) -> None config_data_from_env = os.getenv("PYTHON_GITLAB_CONFIG") if config_data_from_env: # prefer to load config from env variable @@ -59,7 +62,7 @@ class Gitlab(object): self._init_gitlab_inst(project_id, config_files) @retry - def _init_gitlab_inst(self, project_id: Optional[int], config_files: Optional[List[str]]) -> None: + def _init_gitlab_inst(self, project_id, config_files): # type: (Optional[int], Optional[List[str]]) -> None gitlab_id = os.getenv('LOCAL_GITLAB_HTTPS_HOST') # if None, will use the default gitlab server self.gitlab_inst = gitlab.Gitlab.from_config(gitlab_id=gitlab_id, config_files=config_files) self.gitlab_inst.auth() @@ -69,7 +72,7 @@ class Gitlab(object): self.project = None @retry - def get_project_id(self, name: str, namespace: Optional[str] = None) -> int: + def get_project_id(self, name, namespace=None): # type: (str, Optional[str]) -> int """ search project ID by name @@ -96,7 +99,7 @@ class Gitlab(object): return int(res[0]) @retry - def download_artifacts(self, job_id: int, destination: str) -> None: + def download_artifacts(self, job_id, destination): # type (int, str) -> None """ download full job artifacts and extract to destination. @@ -112,7 +115,7 @@ class Gitlab(object): archive_file.extractall(destination) @retry - def download_artifact(self, job_id: int, artifact_path: str, destination: Optional[str] = None) -> List[bytes]: + def download_artifact(self, job_id, artifact_path, destination=None): # type: (int, str, Optional[str]) -> List[bytes] """ download specific path of job artifacts and extract to destination. @@ -145,7 +148,7 @@ class Gitlab(object): return raw_data_list @retry - def find_job_id(self, job_name: str, pipeline_id: Optional[str] = None, job_status: str = 'success') -> List[Dict]: + def find_job_id(self, job_name, pipeline_id=None, job_status='success'): # type: (str, Optional[str], str) -> List[Dict] """ Get Job ID from job name of specific pipeline @@ -168,7 +171,7 @@ class Gitlab(object): return job_id_list @retry - def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None) -> str: + def download_archive(self, ref, destination, project_id=None): # type: (str, str, Optional[int]) -> str """ Download archive of certain commit of a repository and extract to destination path @@ -198,7 +201,7 @@ class Gitlab(object): return os.path.join(os.path.realpath(destination), root_name) -def main() -> None: +def main(): # type: () -> None parser = argparse.ArgumentParser() parser.add_argument("action") parser.add_argument("project_id", type=int)