Merge pull request #405 from espressif/fix/pre-run-failure

Fix/pre run failure

Author: hfudev

.gitignore

@@ -71,7 +71,7 @@ venv/
 .idea/
 
 # built binaries
-tests/fixtures/*/build/*
+tests/fixtures/*/build*/*
 !tests/fixtures/*_coredump_*/build/gdbinit/
 tests/fixtures/*_coredump_*/build/gdbinit/*
 !tests/fixtures/*_coredump_*/build/gdbinit/prefix_map
@@ -82,15 +82,15 @@ tests/fixtures/*_panic/build/gdbinit/*
 !tests/fixtures/*_panic/build/gdbinit/prefix_map
 !tests/fixtures/*_panic/build/*.elf
 
-!tests/fixtures/*/build/bootloader/
-tests/fixtures/*/build/bootloader/*
-!tests/fixtures/*/build/bootloader/*.bin
-!tests/fixtures/*/build/partition_table/
-!tests/fixtures/*/build/config/
-tests/fixtures/*/build/config/*
-!tests/fixtures/*/build/config/sdkconfig.json
-!tests/fixtures/*/build/flasher_args.json
-!tests/fixtures/*/build/*.bin
+!tests/fixtures/*/build*/bootloader/
+tests/fixtures/*/build*/bootloader/*
+!tests/fixtures/*/build*/bootloader/*.bin
+!tests/fixtures/*/build*/partition_table/
+!tests/fixtures/*/build*/config/
+tests/fixtures/*/build*/config/*
+!tests/fixtures/*/build*/config/sdkconfig.json
+!tests/fixtures/*/build*/flasher_args.json
+!tests/fixtures/*/build*/*.bin
 
 sdkconfig
 sdkconfig.old

.idf_build_apps.toml

@@ -0,0 +1,4 @@
+recursive = true
+check_warnings = true
+build_dir = "build_@t"
+verbose = 1  # INFO

pytest-embedded-idf/pytest_embedded_idf/unity_tester.py

@@ -379,9 +379,14 @@ def _run_normal_case(
         try:
             _start_at = self._prepare_and_start_case(case, reset, timeout)
         except Exception as e:
-            self._analyze_test_case_result(case, e)
-        else:
-            self._analyze_test_case_result(case, None, start_time=_start_at, timeout=timeout)
+            logging.debug('pre_run_failure: %s. hard reset and retry', e)
+            try:
+                _start_at = self._prepare_and_start_case(case, True, timeout)
+            except Exception as e2:
+                self._analyze_test_case_result(case, e2)
+                return
+
+        self._analyze_test_case_result(case, None, start_time=_start_at, timeout=timeout)
 
     def _run_multi_stage_case(
         self,
@@ -408,37 +413,42 @@ def _run_multi_stage_case(
         try:
             _start_at = self._prepare_and_start_case(case, reset, timeout)
         except Exception as e:
-            self._analyze_test_case_result(case, e)
-        else:
-            failed_subcases = []
+            logging.debug('pre_run_failure: %s. hard reset and retry', e)
             try:
-                for sub_case in case.subcases:
-                    if sub_case != case.subcases[0]:
-                        ready_before = self._get_ready(timeout, return_before=True)
-                        if ready_before and UNITY_SUMMARY_LINE_REGEX.search(ready_before):
-                            attrs = _parse_unity_test_output(
-                                remove_asci_color_code(ready_before), case.name, self.pexpect_proc.buffer_debug_str
-                            )
-                            if attrs['result'] == 'FAIL':
-                                failed_subcases.append(attrs)
-
-                        self.confirm_write(case.index, expect_str=f'Running {case.name}...')
-
-                    self.write(str(sub_case['index']))
-            except Exception:
-                # Any exception during the sub-case loop is a runtime failure.
-                # We'll stop sending commands and let the result recorder handle the failure.
-                pass
-            finally:
-                attrs = self._read_result_and_parse_attrs(case, _start_at, timeout)
+                _start_at = self._prepare_and_start_case(case, True, timeout)
+            except Exception as e2:
+                self._analyze_test_case_result(case, e2)
+                return
 
-                if attrs['result'] == 'FAIL':
-                    failed_subcases.append(attrs)
+        failed_subcases = []
+        try:
+            for sub_case in case.subcases:
+                if sub_case != case.subcases[0]:
+                    ready_before = self._get_ready(timeout, return_before=True)
+                    if ready_before and UNITY_SUMMARY_LINE_REGEX.search(ready_before):
+                        attrs = _parse_unity_test_output(
+                            remove_asci_color_code(ready_before), case.name, self.pexpect_proc.buffer_debug_str
+                        )
+                        if attrs['result'] == 'FAIL':
+                            failed_subcases.append(attrs)
+
+                    self.confirm_write(case.index, expect_str=f'Running {case.name}...')
+
+                self.write(str(sub_case['index']))
+        except Exception:
+            # Any exception during the sub-case loop is a runtime failure.
+            # We'll stop sending commands and let the result recorder handle the failure.
+            pass
+        finally:
+            attrs = self._read_result_and_parse_attrs(case, _start_at, timeout)
 
-                if failed_subcases:
-                    self._add_test_case_to_suite(self._squash_failed_subcases(failed_subcases, _start_at))
-                else:
-                    self._add_test_case_to_suite(attrs)
+            if attrs['result'] == 'FAIL':
+                failed_subcases.append(attrs)
+
+            if failed_subcases:
+                self._add_test_case_to_suite(self._squash_failed_subcases(failed_subcases, _start_at))
+            else:
+                self._add_test_case_to_suite(attrs)
 
     def run_single_board_case(self, name: str, reset: bool = False, timeout: float = 30) -> None:
         for case in self.test_menu:

pytest-embedded-idf/tests/test_idf.py

@@ -62,8 +62,9 @@ def test_idf_serial_flash(dut):
 def test_custom_idf_device_dut(testdir):
     p = os.path.join(testdir.tmpdir, 'hello_world_esp32')
     p_c3 = os.path.join(testdir.tmpdir, 'hello_world_esp32c3')
-    unity_test_path = os.path.join(testdir.tmpdir, 'unit_test_app_esp32')
-    unity_test_path_c3 = os.path.join(testdir.tmpdir, 'unit_test_app_esp32c3')
+
+    unity_test_app_path = os.path.join(testdir.tmpdir, 'unit_test_app_idf')
+
     testdir.makepyfile(f"""
         import pytest
 
@@ -81,14 +82,20 @@ def test_idf_mixed(dut):
 
         def test_idf_unity_tester():
             from pytest_embedded.dut_factory import DutFactory
-            dut1 = DutFactory.create(embedded_services='esp,idf', app_path=r'{unity_test_path}')
-            dut2 = DutFactory.create(embedded_services='esp,idf', app_path=r'{unity_test_path_c3}')
+            dut1 = DutFactory.create(
+                embedded_services='esp,idf', app_path=r'{unity_test_app_path}', build_dir='build_esp32'
+            )
+            dut2 = DutFactory.create(
+                embedded_services='esp,idf', app_path=r'{unity_test_app_path}', build_dir='build_esp32c3'
+            )
             tester = DutFactory.unity_tester(dut1, dut2)
             tester.run_all_multi_dev_cases(timeout=10)
 
         def test_idf_run_all_single_board_cases():
             from pytest_embedded.dut_factory import DutFactory
-            dut1 = DutFactory.create(embedded_services='esp,idf', app_path=r'{unity_test_path}')
+            dut1 = DutFactory.create(
+                embedded_services='esp,idf', app_path=r'{unity_test_app_path}', build_dir='build_esp32'
+            )
             dut1.run_all_single_board_cases(reset=True, timeout=10)
     """)
 
@@ -424,7 +431,9 @@ def test_flash_another_app(dut):
     result = testdir.runpytest(
         '-s',
         '--app-path',
-        os.path.join(testdir.tmpdir, 'unit_test_app_esp32'),
+        os.path.join(testdir.tmpdir, 'unit_test_app_idf'),
+        '--build-dir',
+        'build_esp32',
         '--embedded-services',
         'esp,idf',
         '--part-tool',
@@ -748,55 +757,20 @@ def test_select_to_run():
     assert IdfUnityDutMixin._select_to_run([['hello', '!w']], None, None, ['hello', 'world'], None, None)
 
 
-def test_dut_run_all_single_board_cases_reset_false(testdir):
-    testdir.makepyfile(r"""
-        def test_dut_run_all_single_board_cases(dut):
-            dut.run_all_single_board_cases(timeout=10)
-    """)
-    testdir.runpytest(
-        '-s',
-        '--embedded-services',
-        'esp,idf',
-        '--app-path',
-        os.path.join(testdir.tmpdir, 'unit_test_app_esp32c3'),
-        '--log-cli-level',
-        'DEBUG',
-        '--junitxml',
-        'report.xml',
-    )
-
-    junit_report = ET.parse('report.xml').getroot()[0]
-
-    assert junit_report.attrib['errors'] == '0'
-    assert junit_report.attrib['failures'] == '2'
-    assert junit_report.attrib['skipped'] == '2'
-    assert junit_report.attrib['tests'] == '1'
-
-    testcases = junit_report.findall('.//testcase')
-    assert testcases[0].attrib['name'] == 'normal_case_pass'
-    assert testcases[1].attrib['name'] == 'normal_case_crash'
-    assert testcases[2].attrib['name'] == 'normal_case_stuck'
-    assert testcases[3].attrib['name'] == 'normal_case_skip_when_not_reset'
-    assert testcases[4].attrib['name'] == 'multiple_stages_test'
-
-    assert 10 < float(testcases[1].attrib['time']) < 10.1
-    assert 'Guru Meditation Error' in testcases[1][0].attrib['message']
-
-    assert 'Skipped due to a failure before test execution.' in testcases[3][0].attrib['message']
-    assert 'Skipped due to a failure before test execution.' in testcases[4][0].attrib['message']
-
-
-def test_dut_run_all_single_board_cases_reset_true(testdir):
-    testdir.makepyfile(r"""
+@pytest.mark.parametrize('reset', [True, False])
+def test_dut_run_all_single_board_cases(testdir, reset):
+    testdir.makepyfile(rf"""
         def test_dut_run_all_single_board_cases(dut):
-            dut.run_all_single_board_cases(reset=True, timeout=10)
+            dut.run_all_single_board_cases(timeout=10, reset={reset})
     """)
     testdir.runpytest(
         '-s',
         '--embedded-services',
         'esp,idf',
         '--app-path',
-        os.path.join(testdir.tmpdir, 'unit_test_app_esp32'),
+        os.path.join(testdir.tmpdir, 'unit_test_app_idf'),
+        '--build-dir',
+        'build_esp32c3',
         '--log-cli-level',
         'DEBUG',
         '--junitxml',
@@ -814,7 +788,7 @@ def test_dut_run_all_single_board_cases(dut):
     assert testcases[0].attrib['name'] == 'normal_case_pass'
     assert testcases[1].attrib['name'] == 'normal_case_crash'
     assert testcases[2].attrib['name'] == 'normal_case_stuck'
-    assert testcases[3].attrib['name'] == 'normal_case_skip_when_not_reset'
+    assert testcases[3].attrib['name'] == 'normal_case_pass_auto_reset'
     assert testcases[4].attrib['name'] == 'multiple_stages_test'
 
     assert 10 < float(testcases[1].attrib['time']) < 10.1
@@ -832,7 +806,9 @@ def test_dut_run_all_single_board_cases(dut):
         '--embedded-services',
         'esp,idf',
         '--app-path',
-        os.path.join(testdir.tmpdir, 'unit_test_app_esp32'),
+        os.path.join(testdir.tmpdir, 'unit_test_app_idf'),
+        '--build-dir',
+        'build_esp32',
         '--log-cli-level',
         'DEBUG',
         '--junitxml',
@@ -843,8 +819,8 @@ def test_dut_run_all_single_board_cases(dut):
 
     assert junit_report.attrib['errors'] == '0'
     assert junit_report.attrib['failures'] == '2'
-    assert junit_report.attrib['skipped'] == '1'
-    assert junit_report.attrib['tests'] == '1'
+    assert junit_report.attrib['skipped'] == '0'
+    assert junit_report.attrib['tests'] == '2'
 
 
 def test_dut_run_all_single_board_cases_invert_group(testdir):
@@ -857,7 +833,9 @@ def test_dut_run_all_single_board_cases(dut):
         '--embedded-services',
         'esp,idf',
         '--app-path',
-        os.path.join(testdir.tmpdir, 'unit_test_app_esp32'),
+        os.path.join(testdir.tmpdir, 'unit_test_app_idf'),
+        '--build-dir',
+        'build_esp32',
         '--junitxml',
         'report.xml',
     )
@@ -882,7 +860,9 @@ def test_dut_run_all_single_board_cases(dut):
         '--embedded-services',
         'esp,idf',
         '--app-path',
-        os.path.join(testdir.tmpdir, 'unit_test_app_esp32'),
+        os.path.join(testdir.tmpdir, 'unit_test_app_idf'),
+        '--build-dir',
+        'build_esp32',
         '--junitxml',
         'report.xml',
     )
@@ -916,9 +896,9 @@ def test_unity_test_case_runner_without_reset(unity_tester):
         '--count',
         2,
         '--app-path',
-        f'{os.path.join(testdir.tmpdir, "unit_test_app_esp32")}'
-        f'|'
-        f'{os.path.join(testdir.tmpdir, "unit_test_app_esp32c3")}',
+        os.path.join(testdir.tmpdir, 'unit_test_app_idf'),
+        '--build-dir',
+        'build_esp32|build_esp32c3',
         '--junitxml',
         'report.xml',
     )
@@ -949,9 +929,9 @@ def test_unity_test_case_runner_without_reset(unity_tester):
         '--count',
         2,
         '--app-path',
-        f'{os.path.join(testdir.tmpdir, "unit_test_app_esp32")}'
-        f'|'
-        f'{os.path.join(testdir.tmpdir, "unit_test_app_esp32c3")}',
+        f'{os.path.join(testdir.tmpdir, "unit_test_app_idf")}',
+        '--build-dir',
+        'build_esp32|build_esp32c3',
         '--junitxml',
         'report.xml',
     )
@@ -1003,7 +983,9 @@ def test_python_case(dut):
         '--embedded-services',
         'esp,idf',
         '--app-path',
-        os.path.join(testdir.tmpdir, 'unit_test_app_esp32c3'),
+        f'{os.path.join(testdir.tmpdir, "unit_test_app_idf")}',
+        '--build-dir',
+        'build_esp32c3',
         '--junitxml',
         'report.xml',
         '--unity-test-report-mode',