blackbeard420/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf synced 2025-03-10 09:39:10 -04:00
Code Issues Projects Releases Wiki Activity

Merge branch 'ci/fix_artifacts_object_name' into 'master'

Browse Source 
ci: fix job name sanitizer

See merge request espressif/esp-idf!28676
This commit is contained in:
Fu Hanxi 2024-01-24 23:08:50 +08:00
commit cba1665639
2 changed files with 22 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
tools/ci/artifacts_handler.py
View File

@ -1,10 +1,9 @@
# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD # SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
import argparse import argparse
import fnmatch import fnmatch
import glob import glob
import os import os
import re
import typing as t import typing as t
import zipfile import zipfile
from enum import Enum from enum import Enum
@ -12,6 +11,7 @@ from pathlib import Path
from zipfile import ZipFile from zipfile import ZipFile
import urllib3 import urllib3
from idf_ci_utils import sanitize_job_name
from idf_pytest.constants import DEFAULT_BUILD_LOG_FILENAME from idf_pytest.constants import DEFAULT_BUILD_LOG_FILENAME
from minio import Minio from minio import Minio
@ -149,8 +149,7 @@ def _upload_files(
try: try:
if has_file: if has_file:
job_name_sanitized = re.sub(r' \[\d+]', '', job_name) obj_name = f'{pipeline_id}/{artifact_type.value}/{sanitize_job_name(job_name)}/{job_id}.zip'
obj_name = f'{pipeline_id}/{artifact_type.value}/{job_name_sanitized}/{job_id}.zip'
print(f'Created archive file: {job_id}.zip, uploading as {obj_name}') print(f'Created archive file: {job_id}.zip, uploading as {obj_name}')
client.fput_object(getenv('IDF_S3_BUCKET'), obj_name, f'{job_id}.zip') client.fput_object(getenv('IDF_S3_BUCKET'), obj_name, f'{job_id}.zip')

26
tools/ci/idf_ci_utils.py
View File

@ -1,11 +1,10 @@
# SPDX-FileCopyrightText: 2020-2023 Espressif Systems (Shanghai) CO LTD # SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# internal use only for CI # internal use only for CI
# some CI related util functions # some CI related util functions
import logging import logging
import os import os
import re
import subprocess import subprocess
import sys import sys
import typing as t import typing as t
@ -178,10 +177,6 @@ class GitlabYmlConfig:
def get_all_manifest_files() -> t.List[str]: def get_all_manifest_files() -> t.List[str]:
"""
:rtype: object
"""
paths: t.List[str] = [] paths: t.List[str] = []
for p in Path(IDF_PATH).glob('**/.build-test-rules.yml'): for p in Path(IDF_PATH).glob('**/.build-test-rules.yml'):
@ -191,3 +186,20 @@ def get_all_manifest_files() -> t.List[str]:
paths.append(str(p)) paths.append(str(p))
return paths return paths
def sanitize_job_name(name: str) -> str:
"""
Sanitize the job name from CI_JOB_NAME
- for job with `parallel: int` set, the `CI_JOB_NAME` would be `job_name index/total`, like `foo 1/3`
- for job with `parallel: matrix` set, the `CI_JOB_NAME` would be `job_name: [var1, var2]`, like `foo: [a, b]`
We consider
- the jobs generated by `parallel: int` as the same job, i.e., we remove the index/total part.
- the jobs generated by `parallel: matrix` as different jobs, so we keep the matrix part.
:param name: job name
:return: sanitized job name
"""
return re.sub(r' \d+/\d+', '', name)