From f0dea7d9ff787ec657254d24fd8ecb55d138dd0f Mon Sep 17 00:00:00 2001
From: Peter Dragun <peter.dragun@espressif.com>
Date: Tue, 21 Mar 2023 15:56:30 +0100
Subject: [PATCH] bug(monitor/ansi_converter): fix output decode detection

---
 tools/idf_monitor.py                           |  8 ++++----
 tools/idf_monitor_base/ansi_color_converter.py | 11 ++++++++---
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/tools/idf_monitor.py b/tools/idf_monitor.py
index ac127549f0..46ae347615 100755
--- a/tools/idf_monitor.py
+++ b/tools/idf_monitor.py
@@ -35,7 +35,7 @@ from typing import Any, List, Optional, Type, Union
 import serial
 import serial.tools.list_ports
 # Windows console stuff
-from idf_monitor_base.ansi_color_converter import get_converter
+from idf_monitor_base.ansi_color_converter import get_ansi_converter
 from idf_monitor_base.argument_parser import get_parser
 from idf_monitor_base.console_parser import ConsoleParser
 from idf_monitor_base.console_reader import ConsoleReader
@@ -91,9 +91,9 @@ class Monitor:
         self.cmd_queue = queue.Queue()  # type: queue.Queue
         self.console = miniterm.Console()
         # if the variable is set ANSI will be printed even if we do not print to terminal
-        sys.stderr = get_converter(sys.stderr, decode_output=True, force_color=force_color)
-        self.console.output = get_converter(self.console.output, force_color=force_color)
-        self.console.byte_output = get_converter(self.console.byte_output, force_color=force_color)
+        sys.stderr = get_ansi_converter(sys.stderr, force_color=force_color)  # type: ignore
+        self.console.output = get_ansi_converter(self.console.output, force_color=force_color)
+        self.console.byte_output = get_ansi_converter(self.console.byte_output, force_color=force_color)
 
         self.elf_file = elf_file or ''
         self.elf_exists = os.path.exists(self.elf_file)
diff --git a/tools/idf_monitor_base/ansi_color_converter.py b/tools/idf_monitor_base/ansi_color_converter.py
index f5cd42e11a..0826e0ef83 100644
--- a/tools/idf_monitor_base/ansi_color_converter.py
+++ b/tools/idf_monitor_base/ansi_color_converter.py
@@ -28,7 +28,7 @@ if os.name == 'nt':
     SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute  # type: ignore
 
 
-def get_converter(orig_output_method=None, force_color=False):
+def get_ansi_converter(orig_output_method=None, force_color=False):
     # type: (Any[TextIO, Optional[TextIOBase]], bool) -> Union[ANSIColorConverter, Optional[TextIOBase]]
     """
     Returns an ANSIColorConverter on Windows and the original output method (orig_output_method) on other platforms.
@@ -53,8 +53,13 @@ class ANSIColorConverter(object):
     def __init__(self, output=None, force_color=False):
         # type: (TextIOBase, bool) -> None
         self.output = output
-        # string stream has to be encoded and then decoded back for proper escape sequence handling
-        self.decode_output = isinstance(output, TextIOBase)
+        # check if output supports writing bytes or if decoding before writing is necessary
+        try:
+            output.write(b'')  # type: ignore
+        except Exception:
+            self.decode_output = True
+        else:
+            self.decode_output = False
         self.handle = GetStdHandle(STD_ERROR_HANDLE if self.output == sys.stderr else STD_OUTPUT_HANDLE)
         self.matched = b''
         self.force_color = force_color  # always print ANSI for colors if true