From 88b40b4bea537e845d8a74815d6da31762112679 Mon Sep 17 00:00:00 2001 From: Roland Dobai Date: Fri, 4 Aug 2023 15:09:37 +0200 Subject: [PATCH 1/3] fix(tools): idf_tools.py tests should keep the original constraint file --- .../test_idf_tools_python_env.py | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/tools/test_idf_tools/test_idf_tools_python_env.py b/tools/test_idf_tools/test_idf_tools_python_env.py index 32761280f3..24bce9d6bf 100644 --- a/tools/test_idf_tools/test_idf_tools_python_env.py +++ b/tools/test_idf_tools/test_idf_tools_python_env.py @@ -231,11 +231,37 @@ class TestCustomPythonPathInstall(BasePythonInstall): class TestCheckPythonDependencies(BasePythonInstall): + + """ + The constraint file name is available as the constraint_file attribute. The content of the file is changed by these + tests. The backup_constraint_file is a temporary file with the content of the original constraint file. This is + kept in order to restore the original content of the constraint file. Keeping the original constraint file is + important for consequent tests which should not download a new one especially when the test was run with a custom + constraint file different from the one on dl.espressif.com. + """ + constraint_file: str + backup_constraint_file: str + + @classmethod + def setUpClass(cls): # type: () -> None + cls.constraint_file = idf_tools.get_constraints(idf_tools.get_idf_version(), online=False) + with tempfile.NamedTemporaryFile() as f: + cls.backup_constraint_file = f.name + shutil.copyfile(cls.constraint_file, cls.backup_constraint_file) + + @classmethod + def tearDownClass(cls): # type: () -> None + try: + os.remove(cls.backup_constraint_file) + except OSError: + pass + def setUp(self): # type: () -> None if os.path.isdir(PYTHON_DIR): shutil.rmtree(PYTHON_DIR) def tearDown(self): # type: () -> None + shutil.copyfile(self.backup_constraint_file, self.constraint_file) if os.path.isdir(PYTHON_DIR): shutil.rmtree(PYTHON_DIR) @@ -256,14 +282,10 @@ class TestCheckPythonDependencies(BasePythonInstall): # are also present in the freeze list. con_list = [r.replace('==', '>') for r in freeze_output.splitlines() if r.split('==')[0] in req_list] - con_fn = idf_tools.get_constraints(idf_tools.get_idf_version(), online=False) - # delete modified constraints file after this test is finished - self.addCleanup(os.remove, con_fn) - # Write the created constraints list into existing constraints file. # It will not be overwritten by subsequent idf_tools.py run, because # there is timestamp check. - with open(con_fn, 'w') as fd: + with open(self.constraint_file, 'w') as fd: fd.write(os.linesep.join(con_list)) # Test that check_python_dependencies reports that requirements are not satisfied for @@ -283,12 +305,8 @@ class TestCheckPythonDependencies(BasePythonInstall): foo_pkg = self.dump_foopackage() self.run_in_venv(['-m', 'pip', 'install', foo_pkg]) - con_fn = idf_tools.get_constraints(idf_tools.get_idf_version(), online=False) - # delete modified constraints file after this test is finished - self.addCleanup(os.remove, con_fn) - # append foopackage constraint to the existing constraints file - with open(con_fn, 'a') as fd: + with open(self.constraint_file, 'a') as fd: fd.write('foopackage>0.99') # check-python-dependencies should not complain about dummy_package @@ -305,12 +323,8 @@ class TestCheckPythonDependencies(BasePythonInstall): foo_pkg = self.dump_foopackage_dev() self.run_in_venv(['-m', 'pip', 'install', foo_pkg]) - con_fn = idf_tools.get_constraints(idf_tools.get_idf_version(), online=False) - # delete modified constraints file after this test is finished - self.addCleanup(os.remove, con_fn) - # append foopackage constraint to the existing constraints file - with open(con_fn, 'r+') as fd: + with open(self.constraint_file, 'r+') as fd: con_lines = fd.readlines() fd.write('foopackage~=0.98') @@ -318,7 +332,7 @@ class TestCheckPythonDependencies(BasePythonInstall): self.assertIn(REQ_SATISFIED, output) # append foopackage dev version constraint to the existing constraints file - with open(con_fn, 'r+') as fd: + with open(self.constraint_file, 'r+') as fd: fd.writelines(con_lines + ['foopackage==0.99.dev0']) output = self.run_idf_tools(['check-python-dependencies']) From 6b5b8a88db42da3823d764a59aebc15416eb152c Mon Sep 17 00:00:00 2001 From: Roland Dobai Date: Mon, 7 Aug 2023 14:15:01 +0200 Subject: [PATCH 2/3] fix(tools): Keep the original Python environment when idf_tools.py tests are run --- tools/test_idf_tools/test_idf_tools_python_env.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/test_idf_tools/test_idf_tools_python_env.py b/tools/test_idf_tools/test_idf_tools_python_env.py index 24bce9d6bf..8a47a00939 100644 --- a/tools/test_idf_tools/test_idf_tools_python_env.py +++ b/tools/test_idf_tools/test_idf_tools_python_env.py @@ -28,6 +28,7 @@ except ImportError: IDF_PATH = os.environ.get('IDF_PATH', '../..') TOOLS_DIR = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(idf_tools.IDF_TOOLS_PATH_DEFAULT) PYTHON_DIR = os.path.join(TOOLS_DIR, 'python_env') +PYTHON_DIR_BACKUP = tempfile.mkdtemp() REQ_SATISFIED = 'Python requirements are satisfied' REQ_CORE = '- {}'.format(os.path.join(IDF_PATH, 'tools', 'requirements', 'requirements.core.txt')) REQ_GDBGUI = '- {}'.format(os.path.join(IDF_PATH, 'tools', 'requirements', 'requirements.gdbgui.txt')) @@ -40,6 +41,17 @@ idf_tools.global_idf_path = IDF_PATH idf_tools.global_idf_tools_path = TOOLS_DIR +def setUpModule(): # type: () -> None + shutil.rmtree(PYTHON_DIR_BACKUP) + shutil.move(PYTHON_DIR, PYTHON_DIR_BACKUP) + + +def tearDownModule(): # type: () -> None + if os.path.isdir(PYTHON_DIR): + shutil.rmtree(PYTHON_DIR) + shutil.move(PYTHON_DIR_BACKUP, PYTHON_DIR) + + class BasePythonInstall(unittest.TestCase): def run_tool(self, cmd): # type: (List[str]) -> str ret = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=300) @@ -262,8 +274,6 @@ class TestCheckPythonDependencies(BasePythonInstall): def tearDown(self): # type: () -> None shutil.copyfile(self.backup_constraint_file, self.constraint_file) - if os.path.isdir(PYTHON_DIR): - shutil.rmtree(PYTHON_DIR) def test_check_python_dependencies(self): # type: () -> None # Prepare artificial constraints file containing packages from From f5c596e85b5d55b49de10558b93b83bdd2fe3935 Mon Sep 17 00:00:00 2001 From: Roland Dobai Date: Mon, 7 Aug 2023 14:45:32 +0200 Subject: [PATCH 3/3] fix(tools): Increase timeout in idf_tools.py Python tests --- tools/test_idf_tools/test_idf_tools_python_env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/test_idf_tools/test_idf_tools_python_env.py b/tools/test_idf_tools/test_idf_tools_python_env.py index 8a47a00939..f9c202a2ca 100644 --- a/tools/test_idf_tools/test_idf_tools_python_env.py +++ b/tools/test_idf_tools/test_idf_tools_python_env.py @@ -54,7 +54,7 @@ def tearDownModule(): # type: () -> None class BasePythonInstall(unittest.TestCase): def run_tool(self, cmd): # type: (List[str]) -> str - ret = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=300) + ret = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=600) decoded_output = ret.stdout.decode('utf-8', 'ignore') with open(os.path.join(IDF_PATH, 'tools', 'test_idf_tools', 'test_python_env_logs.txt'), 'a+') as w: # stack() returns list of callers frame records. [1] represent caller of this function