feat(bt): Add support for converting BT HCI logs to btsnoop format

(cherry picked from commit decb24f940d9d47e4eef7449748c956500d9449c)

Co-authored-by: zhanghaipeng <zhanghaipeng@espressif.com>

examples/bluetooth/bluedroid/ble/gatt_client/main/gattc_demo.c

@@ -517,4 +517,23 @@ void app_main(void)
         ESP_LOGE(GATTC_TAG, "set local  MTU failed, error code = %x", local_mtu_ret);
     }
 
+
+    /*
+    * This code is intended for debugging and prints all HCI data.
+    * To enable it, turn on the "BT_HCI_LOG_DEBUG_EN" configuration option.
+    * The output HCI data can be parsed using the script:
+    * esp-idf/tools/bt/bt_hci_to_btsnoop.py.
+    * For detailed instructions, refer to esp-idf/tools/bt/README.md.
+    */
+
+    /*
+    while (1) {
+        extern void bt_hci_log_hci_data_show(void);
+        extern void bt_hci_log_hci_adv_show(void);
+        bt_hci_log_hci_data_show();
+        bt_hci_log_hci_adv_show();
+        vTaskDelay(1000 / portNUM_PROCESSORS);
+    }
+    */
+
 }

tools/bt/README.md

@@ -0,0 +1,115 @@
+---
+
+### **Usage Instructions**
+
+The **`bt_hci_to_btsnoop.py`** script parses Bluetooth HCI logs and converts them into the BTSnoop format, allowing for detailed analysis. It reads an input log file, processes each line, and writes HCI packets to an output file.
+
+---
+
+### **How to Capture HCI Logs**
+
+To use this tool, you need to enable Bluetooth HCI debug mode in your ESP-IDF project and capture HCI logs as follows:
+
+#### 1. Enable Bluetooth HCI Debug Mode
+- Open the ESP-IDF configuration menu:
+  - **Top → Component config → Bluetooth**
+  - Enable: `[x] Enable Bluetooth HCI debug mode`
+
+#### 2. Capture Logs in the `app_main` Code
+In your application code (e.g., `app_main`), call the following functions to log HCI data and advertisement logs:
+
+```c
+while (1)
+{
+    extern void bt_hci_log_hci_data_show(void);
+    extern void bt_hci_log_hci_adv_show(void);
+    bt_hci_log_hci_data_show();  // Display HCI data logs
+    bt_hci_log_hci_adv_show();   // Display HCI advertisement logs
+    vTaskDelay(1000 / portNUM_PROCESSORS);
+}
+```
+
+#### 3. Save Logs to a File
+Run the application and redirect the serial output to a log file:
+
+```bash
+idf.py -p /dev/ttyUSB0 monitor >> all_log.txt
+```
+
+- **Note:** Replace `/dev/ttyUSB0` with your device's actual port (e.g., `/dev/ttyUSB1`, `COM3`, etc.).
+
+#### 4. Manual Log Creation (Optional)
+Alternatively, manually create a log file containing HCI data in the expected format.
+
+---
+
+### **Running the Script**
+
+To parse the logs and generate a BTSnoop file, run:
+
+```bash
+python bt_hci_to_btsnoop.py -p <input_log_file> -o <output_tag>
+```
+
+#### **Parameters**
+- `-p` or `--path`: Path to the input log file (e.g., `all_log.txt`). Required.
+- `-o` or `--output`: A tag for the output file name. Required.
+
+#### **Example Command**
+```bash
+python bt_hci_to_btsnoop.py -p /path/to/input_log.txt -o 123
+```
+
+This creates the file: `./parsed_logs/parsed_log_123.btsnoop.log`.
+
+#### **Help Information**
+For detailed usage options, run:
+```bash
+python bt_hci_to_btsnoop.py -h
+```
+
+---
+
+### **Generated Output**
+
+The parsed file will be saved as `parsed_log_<output_tag>.btsnoop.log` in the BTSnoop format.
+
+### **Supported Tools for Viewing HCI Logs**
+
+1. **Wireshark**
+   - **Download:** [Wireshark](https://www.wireshark.org/)
+   - Compatible with **Windows**, **Linux**, and **macOS** platforms.
+   - Use it to open `.btsnoop.log` files for detailed analysis of HCI packets.
+
+2. **Wireless Protocol Suite** (Teledyne LeCroy)
+   - **Download:** [Wireless Protocol Suite](https://www.teledynelecroy.com/support/softwaredownload/psgdocuments.aspx?standardid=2&mseries=672)
+   - Supported exclusively on the **Windows** platform.
+
+3. **btmon** (Linux only)
+   - **Description:** A command-line tool included with the BlueZ Bluetooth stack for Linux.
+   - Use the following command to analyze HCI details directly:
+     ```bash
+     btmon -r <btsnoop log>
+     ```
+---
+
+### **Sample Log Format**
+
+Below is an example of expected log entries:
+
+```plaintext
+e6 C:35 0c 05 01 01 00 01 00
+e7 H:01 00 2a 00 26 00 04 00 12 2a 00 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20 21 22
+e8 E:13 05 01 01 00 01 00
+e9 D:01 20 05 00 01 00 04 00 13
+ea C:35 0c 05 01 01 00 01 00
+```
+
+---
+
+### **Notes**
+- Ensure valid input file paths and output directories.
+- Verify read/write permissions for files.
+- The script expects a specific log file format; unexpected formats may cause errors.
+
+By following these steps, you can easily convert HCI logs into the BTSnoop format for analysis.

tools/bt/bt_hci_to_btsnoop.py

@@ -0,0 +1,134 @@
+# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import argparse
+import os
+import re
+import struct
+import time
+
+parsed_num = 0
+all_line_num = 0
+
+
+def create_new_bt_snoop_file(filename: str) -> None:
+    with open(filename, 'wb') as f:
+        magic = b'btsnoop\x00'
+        f.write(magic)
+        version = 1
+        datalink = 1002
+        header = struct.pack('>II', version, datalink)
+        f.write(header)
+
+
+def append_hci_to_bt_snoop_file(filename: str, direction: int, data: str) -> None:
+    if os.path.exists(filename):
+        print(f'Appending to existing file: {filename}')
+    else:
+        print(f'Creating new file: {filename}')
+        create_new_bt_snoop_file(filename)
+
+    # Ensure data is converted to bytearray from hex string
+    data_bytes = bytearray.fromhex(data)  # Convert hex string to bytearray
+
+    with open(filename, 'ab') as f:  # 'ab' mode for appending binary data
+        global parsed_num
+        parsed_num += 1
+        data_len = len(data_bytes)
+        orig_len = data_len
+        incl_len = data_len
+        packet_flags = direction + (data_bytes[0] != 1 or data_bytes[0] != 3) * 2
+        cumulative_drops = 0
+        # Calculate the timestamp with an offset from a predefined reference time(0x00DCDDB30F2F8000).
+        timestamp_us = int(round(time.time() * 1000000)) + 0x00DCDDB30F2F8000
+
+        # Writing structured data followed by the byte array data
+        f.write(struct.pack('>IIIIQ', orig_len, incl_len, packet_flags, cumulative_drops, timestamp_us))
+        f.write(data_bytes)  # Write bytearray (binary data) to file
+
+
+def log_data_clean(data: str) -> str:
+    cleaned = re.sub(r'.*?<pre>', '', data)
+    return cleaned
+
+
+def is_adv_report(data: str) -> bool:
+    is_binary = all(re.fullmatch(r'[0-9a-fA-F]{2}', part) for part in data.split())
+    return is_binary
+
+
+def parse_log(input_path: str, output_tag: str) -> None:
+    """
+    Parses the specified log file and saves the results to an output file.
+
+    Args:
+        input_path (str): Path to the input log file.
+        output_tag (str): Name tag for the output file.
+
+    Returns:
+        None
+    """
+    global parsed_num
+    global all_line_num
+    if not os.path.exists(input_path):
+        print(f"Error: The file '{input_path}' does not exist.")
+        return
+
+    # Define the output directory and create it if it doesn't exist
+    output_dir = './parsed_logs'
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Define the output file name based on the tag
+    output_file = os.path.join(output_dir, f'parsed_log_{output_tag}.btsnoop.log')
+
+    with open(input_path, 'r', encoding='utf-8') as infile:
+        # Example: Parse each line and save processed results
+        for line in infile:
+            try:
+                all_line_num += 1
+                line = log_data_clean(line)
+                line = line.replace('C:', '01 ')
+                line = line.replace('E:', '04 ')
+                line = line[3:]
+                if line[0] == 'H':
+                    line = line.replace('H:', '02 ')
+                    append_hci_to_bt_snoop_file(output_file, 0, line)
+                    continue
+                if line[0] == 'D':
+                    line = line.replace('D:', '02 ')
+                    append_hci_to_bt_snoop_file(output_file, 1, line)
+                    continue
+                if line.startswith('01'):
+                    append_hci_to_bt_snoop_file(output_file, 0, line)
+                    continue
+                if line.startswith('04'):
+                    append_hci_to_bt_snoop_file(output_file, 1, line)
+                    continue
+                if is_adv_report(line):
+                    line = '04 3e ' + line
+                    append_hci_to_bt_snoop_file(output_file, 1, line)
+            except Exception as e:
+                print(f'Exception: {e}')
+
+    if parsed_num > 0:
+        print(f'Log parsing completed, parsed_num {parsed_num}, all_line_num {all_line_num}.\nOutput saved to: {output_file}')
+    else:
+        print('No data could be parsed.')
+
+
+def main() -> None:
+    # Define command-line arguments
+    parser = argparse.ArgumentParser(description='Log Parsing Tool')
+    parser.add_argument('-p', '--path', required=True, help='Path to the input log file')
+    parser.add_argument('-o', '--output', required=True, help='Name tag for the output file')
+
+    # Parse arguments
+    args = parser.parse_args()
+    input_path = args.path
+    output_tag = args.output
+
+    # Call the log parsing function
+    parse_log(input_path, output_tag)
+
+
+if __name__ == '__main__':
+    main()

tools/ci/exclude_check_tools_files.txt

@@ -59,3 +59,5 @@ tools/legacy_exports/export_legacy.sh
 tools/legacy_exports/export_legacy.ps1
 tools/legacy_exports/export_legacy.bat
 tools/ci/idf_build_apps_dump_soc_caps.py
+tools/bt/bt_hci_to_btsnoop.py
+tools/bt/README.md