tools: fixed elf symbols load if gdbinit specified

ROM and bootloader symbols add to use in GDB (via 'idf.py gdb')

components/bootloader/project_include.cmake

@@ -11,8 +11,9 @@ endif()
 #
 idf_build_get_property(build_dir BUILD_DIR)
 set(BOOTLOADER_BUILD_DIR "${build_dir}/bootloader")
+set(BOOTLOADER_ELF_FILE "${BOOTLOADER_BUILD_DIR}/bootloader.elf")
 set(bootloader_binary_files
-    "${BOOTLOADER_BUILD_DIR}/bootloader.elf"
+    "${BOOTLOADER_ELF_FILE}"
     "${BOOTLOADER_BUILD_DIR}/bootloader.bin"
     "${BOOTLOADER_BUILD_DIR}/bootloader.map"
     )

tools/cmake/project_description.json.in

@@ -4,6 +4,7 @@
     "build_dir":          "${BUILD_DIR}",
     "config_file":        "${SDKCONFIG}",
     "config_defaults":    "${SDKCONFIG_DEFAULTS}",
+    "bootloader_elf":     "${BOOTLOADER_ELF_FILE}",
     "app_elf":            "${PROJECT_EXECUTABLE}",
     "app_bin":            "${PROJECT_BIN}",
     "git_revision":       "${IDF_VER}",

tools/idf_py_actions/debug_ext.py

@@ -4,10 +4,12 @@ import json
 import os
 import re
 import shlex
+import shutil
 import subprocess
 import sys
 import threading
 import time
+from textwrap import indent
 from threading import Thread
 from typing import Any, Dict, List, Optional
 
@@ -16,6 +18,45 @@ from idf_py_actions.errors import FatalError
 from idf_py_actions.tools import PropertyDict, ensure_build_directory
 
 PYTHON = sys.executable
+ESP_ROM_INFO_FILE = 'roms.json'
+GDBINIT_PYTHON_TEMPLATE = '''
+# Add Python GDB extensions
+python
+import sys
+sys.path = {sys_path}
+import freertos_gdb
+end
+'''
+GDBINIT_PYTHON_NOT_SUPPORTED = '''
+# Python scripting is not supported in this copy of GDB.
+# Please make sure that your Python distribution contains Python shared library.
+'''
+GDBINIT_BOOTLOADER_ADD_SYMBOLS = '''
+# Load bootloader symbols
+set confirm off
+  add-symbol-file {boot_elf}
+set confirm on
+'''
+GDBINIT_BOOTLOADER_NOT_FOUND = '''
+# Bootloader elf was not found
+'''
+GDBINIT_APP_ADD_SYMBOLS = '''
+# Load application file
+file {app_elf}
+'''
+GDBINIT_CONNECT = '''
+# Connect to the default openocd-esp port and break on app_main()
+target remote :3333
+monitor reset halt
+flushregs
+thbreak app_main
+continue
+'''
+GDBINIT_MAIN = '''
+source {py_extensions}
+source {symbols}
+source {connect}
+'''
 
 
 def action_extensions(base_actions: Dict, project_path: str) -> Dict:
@@ -91,22 +132,111 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict:
         # execute simple python command to check is it supported
         return subprocess.run([gdb, '--batch-silent', '--ex', 'python import os'], stderr=subprocess.DEVNULL).returncode == 0
 
-    def create_local_gdbinit(gdb: str, gdbinit: str, elf_file: str) -> None:
+    def get_normalized_path(path: str) -> str:
-        with open(gdbinit, 'w') as f:
+        if os.name == 'nt':
+            return os.path.normpath(path).replace('\\','\\\\')
+        return path
+
+    def get_rom_if_condition_str(date_addr: int, date_str: str) -> str:
+        r = []
+        for i in range(0, len(date_str), 4):
+            value = hex(int.from_bytes(bytes(date_str[i:i + 4], 'utf-8'), 'little'))
+            r.append(f'(*(int*) {hex(date_addr + i)}) == {value}')
+        return 'if ' + ' && '.join(r)
+
+    def generate_gdbinit_rom_add_symbols(target: str) -> str:
+        base_ident = '  '
+        rom_elfs_dir = os.getenv('ESP_ROM_ELF_DIR')
+        if not rom_elfs_dir:
+            raise FatalError('ESP_ROM_ELF_DIR environment variable is not defined. Please try to run IDF "install" and "export" scripts.')
+        with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), ESP_ROM_INFO_FILE), 'r') as f:
+            roms = json.load(f)
+            if target not in roms:
+                msg_body = f'Target "{target}" was not found in "{ESP_ROM_INFO_FILE}". Please check IDF integrity.'
+                if os.getenv('ESP_IDF_GDB_TESTING'):
+                    raise FatalError(msg_body)
+                print(f'Warning: {msg_body}')
+                return f'# {msg_body}'
+            r = ['', f'# Load {target} ROM ELF symbols']
+            is_one_revision = len(roms[target]) == 1
+            if not is_one_revision:
+                r.append('define target hookpost-remote')
+            r.append('set confirm off')
+            # Workaround for reading ROM data on xtensa chips
+            # This should be deleted after the new openocd-esp release (newer than v0.11.0-esp32-20220706)
+            xtensa_chips = ['esp32', 'esp32s2', 'esp32s3']
+            if target in xtensa_chips:
+                r.append('monitor xtensa set_permissive 1')
+            # Since GDB does not have 'else if' statement than we use nested 'if..else' instead.
+            for i, k in enumerate(roms[target], 1):
+                indent_str = base_ident * i
+                rom_file = get_normalized_path(os.path.join(rom_elfs_dir, f'{target}_rev{k["rev"]}_rom.elf'))
+                build_date_addr = int(k['build_date_str_addr'], base=16)
+                r.append(indent(f'# if $_streq((char *) {hex(build_date_addr)}, "{k["build_date_str"]}")', indent_str))
+                r.append(indent(get_rom_if_condition_str(build_date_addr, k['build_date_str']), indent_str))
+                r.append(indent(f'add-symbol-file {rom_file}', indent_str + base_ident))
+                r.append(indent('else', indent_str))
+                if i == len(roms[target]):
+                    # In case no one known ROM ELF fits - print error and exit with error code 1
+                    indent_str += base_ident
+                    msg_body = f'unknown {target} ROM revision.'
+                    if os.getenv('ESP_IDF_GDB_TESTING'):
+                        r.append(indent(f'echo Error: {msg_body}\\n', indent_str))
+                        r.append(indent('quit 1', indent_str))
+                    else:
+                        r.append(indent(f'echo Warning: {msg_body}\\n', indent_str))
+            # Close 'else' operators
+            for i in range(len(roms[target]), 0, -1):
+                r.append(indent('end', base_ident * i))
+            if target in xtensa_chips:
+                r.append('monitor xtensa set_permissive 0')
+            r.append('set confirm on')
+            if not is_one_revision:
+                r.append('end')
+            r.append('')
+            return os.linesep.join(r)
+        raise FatalError(f'{ESP_ROM_INFO_FILE} file not found. Please check IDF integrity.')
+
+    def generate_gdbinit_files(gdb: str, gdbinit: Optional[str], project_desc: Dict[str, Any]) -> None:
+        app_elf = get_normalized_path(os.path.join(project_desc['build_dir'], project_desc['app_elf']))
+        if not os.path.exists(app_elf):
+            raise FatalError('ELF file not found. You need to build & flash the project before running debug targets')
+
+        # Recreate empty 'gdbinit' directory
+        gdbinit_dir = os.path.join(project_desc['build_dir'], 'gdbinit')
+        if os.path.isfile(gdbinit_dir):
+            os.remove(gdbinit_dir)
+        elif os.path.isdir(gdbinit_dir):
+            shutil.rmtree(gdbinit_dir)
+        os.mkdir(gdbinit_dir)
+
+        # Prepare gdbinit for Python GDB extensions import
+        py_extensions = os.path.join(gdbinit_dir, 'py_extensions')
+        with open(py_extensions, 'w') as f:
             if is_gdb_with_python(gdb):
-                f.write('python\n')
+                f.write(GDBINIT_PYTHON_TEMPLATE.format(sys_path=sys.path))
-                f.write('import sys\n')
+            else:
-                f.write(f'sys.path = {sys.path}\n')
+                f.write(GDBINIT_PYTHON_NOT_SUPPORTED)
-                f.write('import freertos_gdb\n')
+
-                f.write('end\n')
+        # Prepare gdbinit for related ELFs symbols load
-            if os.name == 'nt':
+        symbols = os.path.join(gdbinit_dir, 'symbols')
-                elf_file = elf_file.replace('\\','\\\\')
+        with open(symbols, 'w') as f:
-            f.write('file {}\n'.format(elf_file))
+            boot_elf = get_normalized_path(project_desc['bootloader_elf']) if 'bootloader_elf' in project_desc else None
-            f.write('target remote :3333\n')
+            if boot_elf and os.path.exists(boot_elf):
-            f.write('mon reset halt\n')
+                f.write(GDBINIT_BOOTLOADER_ADD_SYMBOLS.format(boot_elf=boot_elf))
-            f.write('flushregs\n')
+            else:
-            f.write('thb app_main\n')
+                f.write(GDBINIT_BOOTLOADER_NOT_FOUND)
-            f.write('c\n')
+            f.write(generate_gdbinit_rom_add_symbols(project_desc['target']))
+            f.write(GDBINIT_APP_ADD_SYMBOLS.format(app_elf=app_elf))
+
+        # Generate the gdbinit for target connect if no custom gdbinit is present
+        if not gdbinit:
+            gdbinit = os.path.join(gdbinit_dir, 'connect')
+            with open(gdbinit, 'w') as f:
+                f.write(GDBINIT_CONNECT)
+
+        with open(os.path.join(gdbinit_dir, 'gdbinit'), 'w') as f:
+            f.write(GDBINIT_MAIN.format(py_extensions=py_extensions, symbols=symbols, connect=gdbinit))
 
     def debug_cleanup() -> None:
         print('cleaning up debug targets')
@@ -191,7 +321,8 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict:
         processes['openocd_outfile_name'] = openocd_out_name
         print('OpenOCD started as a background task {}'.format(process.pid))
 
-    def get_gdb_args(gdbinit: str, project_desc: Dict[str, Any]) -> List:
+    def get_gdb_args(project_desc: Dict[str, Any]) -> List:
+        gdbinit = os.path.join(project_desc['build_dir'], 'gdbinit', 'gdbinit')
         args = ['-x={}'.format(gdbinit)]
         debug_prefix_gdbinit = project_desc.get('debug_prefix_map_gdbinit')
         if debug_prefix_gdbinit:
@@ -205,16 +336,14 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict:
         project_desc = get_project_desc(args, ctx)
         local_dir = project_desc['build_dir']
         gdb = project_desc['monitor_toolprefix'] + 'gdb'
-        if gdbinit is None:
+        generate_gdbinit_files(gdb, gdbinit, project_desc)
-            gdbinit = os.path.join(local_dir, 'gdbinit')
-            create_local_gdbinit(gdb, gdbinit, os.path.join(args.build_dir, project_desc['app_elf']))
 
         # this is a workaround for gdbgui
         # gdbgui is using shlex.split for the --gdb-args option. When the input is:
         # - '"-x=foo -x=bar"', would return ['foo bar']
         # - '-x=foo', would return ['-x', 'foo'] and mess up the former option '--gdb-args'
         # so for one item, use extra double quotes. for more items, use no extra double quotes.
-        gdb_args_list = get_gdb_args(gdbinit, project_desc)
+        gdb_args_list = get_gdb_args(project_desc)
         gdb_args = '"{}"'.format(' '.join(gdb_args_list)) if len(gdb_args_list) == 1 else ' '.join(gdb_args_list)
         args = ['gdbgui', '-g', gdb, '--gdb-args', gdb_args]
         print(args)
@@ -286,16 +415,9 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict:
         watch_openocd.start()
         processes['threads_to_join'].append(watch_openocd)
         project_desc = get_project_desc(args, ctx)
-
-        elf_file = os.path.join(args.build_dir, project_desc['app_elf'])
-        if not os.path.exists(elf_file):
-            raise FatalError('ELF file not found. You need to build & flash the project before running debug targets', ctx)
         gdb = project_desc['monitor_toolprefix'] + 'gdb'
-        local_dir = project_desc['build_dir']
+        generate_gdbinit_files(gdb, gdbinit, project_desc)
-        if gdbinit is None:
+        args = [gdb, *get_gdb_args(project_desc)]
-            gdbinit = os.path.join(local_dir, 'gdbinit')
-            create_local_gdbinit(gdb, gdbinit, elf_file)
-        args = [gdb, *get_gdb_args(gdbinit, project_desc)]
         if gdb_tui is not None:
             args += ['-tui']
         t = Thread(target=run_gdb, args=(args,))

tools/idf_py_actions/roms.json

@@ -0,0 +1,47 @@
+{
+  "esp32": [
+    {
+      "rev": 0,
+      "build_date_str_addr": "0x3ff9ea80",
+      "build_date_str": "Jun  8 2016"
+    },
+    {
+      "rev": 3,
+      "build_date_str_addr": "0x3ff9e986",
+      "build_date_str": "Jul 29 2019"
+    }
+  ],
+  "esp32s2": [
+    {
+      "rev": 0,
+      "build_date_str_addr": "0x3ffaf34b",
+      "build_date_str": "Oct 25 2019"
+    }
+  ],
+  "esp32s3": [
+    {
+      "rev": 0,
+      "build_date_str_addr": "0x3ff194ad",
+      "build_date_str": "Mar  1 2021"
+    }
+  ],
+  "esp32c2": [
+    {
+      "rev": 0,
+      "build_date_str_addr": "0x3ff47874",
+      "build_date_str": "Jan 27 2022"
+    }
+  ],
+  "esp32c3": [
+    {
+      "rev": 0,
+      "build_date_str_addr": "0x3ff1b878",
+      "build_date_str": "Sep 18 2020"
+    },
+    {
+      "rev": 3,
+      "build_date_str_addr": "0x3ff1a374",
+      "build_date_str": "Feb  7 2021"
+    }
+  ]
+}

tools/idf_py_actions/roms_schema.json

@@ -0,0 +1,29 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "patternProperties": {
+    "^esp32.*$": {
+      "anyOf": [
+        {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "rev": {
+                "type": "integer",
+                "minimum": 0,
+                "description": "Chip revision/ROM revision number"
+              },
+              "build_date_str_addr": {
+                "type": "string",
+                "description": "The ROM build date string address to compare between ROM elf file and chip ROM memory",
+                "pattern": "^0x[0-9a-fA-F]{8}$"
+              }
+            },
+            "required": ["rev", "build_date_str_addr"]
+          }
+        }
+      ]
+    }
+  }
+}

tools/test_idf_py/test_idf_py.py

@@ -9,7 +9,9 @@ import subprocess
 import sys
 from unittest import TestCase, main, mock
 
+import elftools.common.utils as ecu
 import jsonschema
+from elftools.elf.elffile import ELFFile
 
 try:
     from StringIO import StringIO
@@ -25,7 +27,8 @@ except ImportError:
 current_dir = os.path.dirname(os.path.realpath(__file__))
 idf_py_path = os.path.join(current_dir, '..', 'idf.py')
 extension_path = os.path.join(current_dir, 'test_idf_extensions', 'test_ext')
-link_path = os.path.join(current_dir, '..', 'idf_py_actions', 'test_ext')
+py_actions_path = os.path.join(current_dir, '..', 'idf_py_actions')
+link_path = os.path.join(py_actions_path, 'test_ext')
 
 
 class TestWithoutExtensions(TestCase):
@@ -246,5 +249,47 @@ class TestHelpOutput(TestWithoutExtensions):
         action_test(['idf.py', 'help', '--json', '--add-options'], schema_json)
 
 
+class TestROMs(TestWithoutExtensions):
+    def get_string_from_elf_by_addr(self, filename: str, address: int) -> str:
+        result = ''
+        with open(filename, 'rb') as stream:
+            elf_file = ELFFile(stream)
+            ro = elf_file.get_section_by_name('.rodata')
+            ro_addr_delta = ro['sh_addr'] - ro['sh_offset']
+            cstring = ecu.parse_cstring_from_stream(ro.stream, address - ro_addr_delta)
+            if cstring:
+                result = str(cstring.decode('utf-8'))
+        return result
+
+    def test_roms_validate_json(self):
+        with open(os.path.join(py_actions_path, 'roms.json'), 'r') as f:
+            roms_json = json.load(f)
+
+        with open(os.path.join(py_actions_path, 'roms_schema.json'), 'r') as f:
+            schema_json = json.load(f)
+        jsonschema.validate(roms_json, schema_json)
+
+    def test_roms_check_supported_chips(self):
+        from idf_py_actions.constants import SUPPORTED_TARGETS
+        with open(os.path.join(py_actions_path, 'roms.json'), 'r') as f:
+            roms_json = json.load(f)
+        for chip in SUPPORTED_TARGETS:
+            self.assertTrue(chip in roms_json, msg=f'Have no ROM data for chip {chip}')
+
+    def test_roms_validate_build_date(self):
+        sys.path.append(py_actions_path)
+
+        rom_elfs_dir = os.getenv('ESP_ROM_ELF_DIR')
+        with open(os.path.join(py_actions_path, 'roms.json'), 'r') as f:
+            roms_json = json.load(f)
+
+        for chip in roms_json:
+            for k in roms_json[chip]:
+                rom_file = os.path.join(rom_elfs_dir, f'{chip}_rev{k["rev"]}_rom.elf')
+                build_date_str = self.get_string_from_elf_by_addr(rom_file, int(k['build_date_str_addr'], base=16))
+                self.assertTrue(len(build_date_str) == 11)
+                self.assertTrue(build_date_str == k['build_date_str'])
+
+
 if __name__ == '__main__':
     main()