From 5ecca75562a2a6364fa5e5d63fdc693745d3143a Mon Sep 17 00:00:00 2001 From: Jacob Bundgaard Date: Thu, 13 Aug 2026 11:53:56 +0200 Subject: [PATCH 1/2] {Output} Add failing tests for UnicodeEncodeError fallback discarding supported characters `OutputProducer.out` falls back to `output.encode('ascii', 'ignore')` when the destination stream cannot encode the output. Because the retry targets ASCII rather than the stream's own encoding, a single unrepresentable character anywhere in the document discards every non-ASCII character in it, including characters the destination encoding represents perfectly well. The first test writes a payload of `ae oe aa em-dash infinity` to a cp1252 stream. cp1252 supports all of those except U+221E INFINITY, but the current fallback drops the four supported characters along with the unrepresentable one. The second test guards the property that the fallback must not turn valid JSON into something a parser rejects, including for characters outside the Basic Multilingual Plane such as emoji. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WfoMDHj25s5BDVD4HacTd5 --- tests/test_output.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/tests/test_output.py b/tests/test_output.py index 332adb7..79967c3 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -6,10 +6,11 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +import json import unittest from unittest import mock from collections import OrderedDict -from io import StringIO +from io import BytesIO, StringIO, TextIOWrapper from knack.output import OutputProducer, format_json, format_json_color, format_yaml, format_yaml_color, \ format_table, format_tsv @@ -93,6 +94,41 @@ def test_out_json_non_ASCII(self): } """)) + def test_out_json_non_ASCII_unencodable(self): + """ + When the destination stream cannot represent every character, only the characters that its + encoding genuinely cannot represent should be affected. Characters the encoding does support + must survive, and the unrepresentable ones should degrade to something visible. + """ + output_producer = OutputProducer(cli_ctx=self.mock_ctx) + # cp1252 represents æ, ø, å and the em dash, but not U+221E INFINITY. + out_file = TextIOWrapper(BytesIO(), encoding='cp1252') + output_producer.out(CommandResultItem({'contents': 'æ ø å — ∞'}), + formatter=format_json, out_file=out_file) + out_file.flush() + written = out_file.buffer.getvalue().decode('cp1252') + + self.assertEqual(normalize_newlines(written), normalize_newlines( + """{ + "contents": "æ ø å — ?" +} +""")) + + def test_out_json_non_ASCII_unencodable_stays_parseable(self): + """ + The fallback must not turn valid JSON into something a parser rejects, including for + characters outside the Basic Multilingual Plane such as emoji. + """ + output_producer = OutputProducer(cli_ctx=self.mock_ctx) + out_file = TextIOWrapper(BytesIO(), encoding='cp1252') + output_producer.out(CommandResultItem({'contents': 'æ ø å 😀'}), + formatter=format_json, out_file=out_file) + out_file.flush() + written = out_file.buffer.getvalue().decode('cp1252') + + # The characters cp1252 supports survive, and the document still parses. + self.assertEqual(json.loads(written)['contents'], 'æ ø å ?') + # YAML output tests def test_out_yaml_valid(self): From 749d932c8fb4cf286ac2761b11cd893430a06214 Mon Sep 17 00:00:00 2001 From: Jacob Bundgaard Date: Thu, 13 Aug 2026 11:54:56 +0200 Subject: [PATCH 2/2] {Output} Encode the UnicodeEncodeError fallback with the stream's own encoding When the destination stream could not encode the output, the fallback re-encoded the whole document as ASCII with `errors='ignore'`. Because the retry targeted ASCII rather than the stream's actual encoding, one unrepresentable character anywhere in the output silently discarded every non-ASCII character in the whole document, including characters the destination encoding represents perfectly well. On a Windows console in a non-UTF-8 locale, a single emoji or CJK character in a result set stripped every accented Latin character from the rest of it. The fallback now encodes with the stream's own encoding, so only genuinely unrepresentable characters are affected. It uses `replace`, which substitutes '?' and therefore keeps the document parseable in every output format. The alternative, `backslashreplace`, would preserve more information, but it emits `\U0001f600` for characters outside the Basic Multilingual Plane, and since JSON permits only `\uXXXX` that would make emoji-containing output fail to parse. Trading silent corruption for a parse failure seemed the wrong trade for a path shared by every command of every knack-based CLI. The warning text is corrected to match: it previously named `out_file.encoding`, which the fallback did not actually use, and claimed that only unsupported characters were discarded when supported ones were discarded too. Also drops the `.decode('utf-8', 'ignore')` round trip, which was a no-op on bytes that had just been produced by `.encode('ascii', 'ignore')`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WfoMDHj25s5BDVD4HacTd5 --- knack/output.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/knack/output.py b/knack/output.py index ed3bf2a..0015ec7 100644 --- a/knack/output.py +++ b/knack/output.py @@ -152,9 +152,13 @@ def out(self, obj, formatter=None, out_file=None): else: raise except UnicodeEncodeError: - logger.warning("Unable to encode the output with %s encoding. Unsupported characters are discarded.", - out_file.encoding) - print(output.encode('ascii', 'ignore').decode('utf-8', 'ignore'), + # Retry with the stream's own encoding so that characters it *can* represent survive. + # Encoding to 'ascii' here would discard every non-ASCII character in the document, + # not just the ones the destination cannot represent. + encoding = out_file.encoding or 'ascii' + logger.warning("Unable to encode some characters with %s encoding. " + "They are replaced with '?'.", encoding) + print(output.encode(encoding, 'replace').decode(encoding), file=out_file, end='') def get_formatter(self, format_type):