From bad4d3d7a839a76c9dc3d30a587e40e2c378f737 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Mon, 24 Aug 2026 15:50:43 -0400 Subject: [PATCH 1/6] Write CLI output as UTF-8 regardless of the locale encoding Every command that prints lexicon content wrote to a stdout carrying the platform's locale encoding whenever it was not a console -- cp1252 on Windows, ASCII under a C/POSIX locale -- so a character the codepage could not represent raised UnicodeEncodeError and killed the command mid-output. `validate > out.txt` died on an NFD range-element id from a committed fixture, and `export` left a truncated file whose bytes were neither UTF-8 nor a complete export, while the same run through -o was both. main() now reconfigures stdout and stderr to UTF-8 unless they already carry it, leaving a stream that is not a TextIOWrapper untouched. Errors stay strict: the only content UTF-8 cannot encode is a lone surrogate, and no file can carry one into the CLI. export instead builds its own wrapper over the stdout byte layer, with the encoding and newline="" that --output has always opened with. csv writes CRLF row terminators, and a stdout that translates newlines doubled the CR into a blank row between every data row, so a redirected export was not the file -o wrote even when it completed; the two are now byte-identical. Tests drive each path through a byte sink under cp1252 and ASCII codecs, spelling out the newline translation a Windows stream applies so the doubling is reproducible on any platform. Co-authored-by: Claude Opus 5 (1M context) --- CHANGELOG.md | 12 ++++++++++ docs/en/guides/cli.md | 2 ++ src/sil_lift/_cli.py | 43 +++++++++++++++++++++++++++++---- tests/test_cli.py | 55 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec04d9b..f15d905 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,18 @@ releases may contain breaking changes. ## [Unreleased] +### Fixed + +- CLI output is UTF-8 whether or not it is redirected. stdout and stderr were + left on the locale encoding when they were not a console — cp1252 on Windows, + ASCII under a C/POSIX locale — so a character it could not represent raised + `UnicodeEncodeError` and killed the command mid-output: `sil-lift validate + ... > out.txt` died on an NFD range-element id, and `export` left a truncated + file whose bytes were neither UTF-8 nor complete. +- `export` to stdout writes the same bytes as `-o`. csv's CRLF row terminator + was doubled by a stdout that translates newlines, putting a blank row between + every data row on Windows. + ## [0.1.0] - 2026-07-TBD ### Added diff --git a/docs/en/guides/cli.md b/docs/en/guides/cli.md index 7953a37..bbd9443 100644 --- a/docs/en/guides/cli.md +++ b/docs/en/guides/cli.md @@ -67,4 +67,6 @@ senses: 4541 $ sil-lift export dictionary.lift --langs en,fr -o dictionary.csv ``` +All output is UTF-8, on every platform and whether it goes to a console, a pipe, or a `>` redirect — never the locale encoding (cp1252 on Windows, ASCII under a C/POSIX locale), which cannot represent LIFT content. `sil-lift export dictionary.lift > dictionary.csv` therefore writes exactly the bytes `-o dictionary.csv` writes, CRLF row terminators included. + Exit codes: `0` success (warnings allowed, unless `--strict`), `1` findings (validation errors / missing media / warnings under `--strict`), `2` unreadable input. diff --git a/src/sil_lift/_cli.py b/src/sil_lift/_cli.py index e72c083..771fc79 100644 --- a/src/sil_lift/_cli.py +++ b/src/sil_lift/_cli.py @@ -12,7 +12,9 @@ from __future__ import annotations import argparse +import codecs import csv +import io import json import sys import tempfile @@ -230,10 +232,19 @@ def _cmd_export(args: argparse.Namespace) -> int: header.extend([f"gloss_{lang}", f"definition_{lang}"]) out_file: TextIO - if args.output is None: - out_file = sys.stdout - else: + stdout_wrapper: io.TextIOWrapper | None = None + if args.output is not None: out_file = args.output.open("w", encoding="utf-8", newline="") + elif isinstance(sys.stdout, io.TextIOWrapper): + # csv writes CRLF row terminators, and a stdout that translates + # newlines doubles the CR into a blank row between every data row. + # Writing through an explicit wrapper gives a redirected run the + # same bytes --output writes, instead of platform-dependent ones. + out_file = stdout_wrapper = io.TextIOWrapper( + sys.stdout.buffer, encoding="utf-8", newline="" + ) + else: # a replaced stdout need not have a byte layer to wrap + out_file = sys.stdout try: writer = csv.writer(out_file, delimiter="\t" if args.tsv else ",") writer.writerow(header) @@ -249,12 +260,36 @@ def _cmd_export(args: argparse.Namespace) -> int: row.append(_text_or_empty(sense.definition.get(lang))) writer.writerow(row) finally: - if args.output is not None: + out_file.flush() + if stdout_wrapper is not None: + stdout_wrapper.detach() # leave sys.stdout usable + elif args.output is not None: out_file.close() return 0 +def _force_utf8(stream: TextIO) -> None: + """Make one of the standard streams write UTF-8, whatever the locale is. + + A stream that is not a console gets the locale encoding — cp1252 on + Windows, ASCII under a C/POSIX locale — which cannot hold LIFT content, so + one unrepresentable character killed the command mid-output. ``-o`` has + always forced UTF-8; this makes a redirect agree with it. + + Errors stay strict: the only content UTF-8 cannot encode is a lone + surrogate, and no file can carry one into the CLI — both readers reject it, + and the finding that reports one escapes it anyway. + """ + if not isinstance(stream, io.TextIOWrapper): # a replaced stream may be anything + return + if codecs.lookup(stream.encoding).name == "utf-8": + return + stream.reconfigure(encoding="utf-8") + + def main(argv: Sequence[str] | None = None) -> int: + _force_utf8(sys.stdout) + _force_utf8(sys.stderr) parser = argparse.ArgumentParser( prog="sil-lift", description="Utilities for LIFT 0.13 lexicon files.", diff --git a/tests/test_cli.py b/tests/test_cli.py index 28c24bb..0722bf1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,6 +2,8 @@ import io import json import shutil +import sys +import unicodedata from pathlib import Path import pytest @@ -11,6 +13,18 @@ CORPUS_DIR = Path(__file__).parent / "corpus" +def _redirect(monkeypatch: pytest.MonkeyPatch, name: str, encoding: str) -> io.BytesIO: + """Stand in for a redirected standard stream: a byte sink under a locale codec. + + ``newline="\r\n"`` is what a Windows stream does with no newline argument, + spelled explicitly so the CRLF-doubling regression is testable anywhere. + """ + raw = io.BytesIO() + stream = io.TextIOWrapper(raw, encoding=encoding, newline="\r\n") + monkeypatch.setattr(sys, name, stream) + return raw + + def test_validate_clean_file(capsys: pytest.CaptureFixture[str]) -> None: assert main(["validate", str(CORPUS_DIR / "ranges" / "test20080407.lift")]) == 0 out = capsys.readouterr().out @@ -282,3 +296,44 @@ def test_export_filename_with_space(tmp_path: Path) -> None: out = tmp_path / "out.csv" assert main(["export", str(path), "-o", str(out)]) == 0 assert out.is_file() + + +def test_validate_text_output_survives_a_cp1252_stdout(monkeypatch: pytest.MonkeyPatch) -> None: + raw = _redirect(monkeypatch, "stdout", "cp1252") + assert main(["validate", str(CORPUS_DIR / "negative" / "nfd-range-ids.lift")]) == 1 + sys.stdout.flush() + out = raw.getvalue().decode("utf-8") + assert unicodedata.normalize("NFD", "Órfão") in out # the id cp1252 cannot hold + assert out.splitlines()[-1].endswith("warning(s)") # ran to the summary, not truncated + + +def test_export_to_a_locale_stdout_matches_the_output_flag( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = CORPUS_DIR / "spec-examples" / "0.13" / "multiple-forms.lift" + to_file = tmp_path / "out.csv" + assert main(["export", str(source), "-o", str(to_file)]) == 0 + + raw = _redirect(monkeypatch, "stdout", "ascii") + assert main(["export", str(source)]) == 0 + assert raw.getvalue() == to_file.read_bytes() + assert "เอว" in raw.getvalue().decode("utf-8") # a gloss ascii cannot hold + + +def test_error_message_survives_an_ascii_stderr(monkeypatch: pytest.MonkeyPatch) -> None: + raw = _redirect(monkeypatch, "stderr", "ascii") + assert main(["validate", str(CORPUS_DIR / "Órfão.lift")]) == 2 + sys.stderr.flush() + assert "Órfão.lift" in raw.getvalue().decode("utf-8") + + +def test_a_stdout_that_is_not_a_text_wrapper_is_left_alone( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A replaced stdout has no encoding to fix and no byte layer to wrap.""" + sink = io.StringIO() + monkeypatch.setattr(sys, "stdout", sink) + assert main(["export", str(CORPUS_DIR / "spec-examples" / "0.13" / "full-entry.lift")]) == 0 + assert main(["validate", str(CORPUS_DIR / "ranges" / "test20080407.lift")]) == 0 + assert "entry_id" in sink.getvalue() + assert "0 error(s), 0 warning(s)" in sink.getvalue() From 8f90d971037c206658300b03e0b149a39b865612 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Mon, 24 Aug 2026 15:54:27 -0400 Subject: [PATCH 2/6] Drop the changelog entry for the CLI output encoding 0.1.0 has not shipped, so redirected output crashing on the locale encoding, and the doubled CRLF row terminators that went with it, were never released behaviour for a `[Unreleased]` section to record a change against. Co-authored-by: Claude Opus 5 (1M context) --- CHANGELOG.md | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f15d905..ec04d9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,18 +18,6 @@ releases may contain breaking changes. ## [Unreleased] -### Fixed - -- CLI output is UTF-8 whether or not it is redirected. stdout and stderr were - left on the locale encoding when they were not a console — cp1252 on Windows, - ASCII under a C/POSIX locale — so a character it could not represent raised - `UnicodeEncodeError` and killed the command mid-output: `sil-lift validate - ... > out.txt` died on an NFD range-element id, and `export` left a truncated - file whose bytes were neither UTF-8 nor complete. -- `export` to stdout writes the same bytes as `-o`. csv's CRLF row terminator - was doubled by a stdout that translates newlines, putting a blank row between - every data row on Windows. - ## [0.1.0] - 2026-07-TBD ### Added From 6a8680ba99918076ce381a3cea41c167e72a5e98 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 25 Aug 2026 11:36:51 -0400 Subject: [PATCH 3/6] Keep each stream's error handler, and stop owning stdout in export `reconfigure(encoding=...)` resets `errors` to strict unless it is passed one, and neither standard stream defaults to strict: stderr is backslashreplace so a message always arrives, and stdout is surrogateescape on some platforms, which is what round-trips a filename the filesystem encoding could not decode -- `check-media` prints those with no repr escaping, and a lone surrogate is the one thing UTF-8 cannot encode either. Forcing UTF-8 now passes the handler the stream already has, so only the encoding changes. The CSV sink for stdout was a TextIOWrapper over its byte layer, which owns that buffer until detached. An unguarded flush() ran first in the finally block, so a write that failed -- a broken pipe, `| head` -- skipped the detach and left the wrapper to close sys.stdout.buffer from its finalizer, and skipped close() on the --output file for the same reason. A four-line sink encoding to the buffer replaces it: csv needs only write(), nothing owns a borrowed stream, and the cleanup is just closing the file this function opened. stdout is flushed before its bytes go out so nothing of its own sits behind them. Co-authored-by: Claude Opus 5 (1M context) --- src/sil_lift/_cli.py | 57 +++++++++++++++++++++++++++----------------- tests/test_cli.py | 32 ++++++++++++++++++++++--- 2 files changed, 64 insertions(+), 25 deletions(-) diff --git a/src/sil_lift/_cli.py b/src/sil_lift/_cli.py index 771fc79..f7d80af 100644 --- a/src/sil_lift/_cli.py +++ b/src/sil_lift/_cli.py @@ -30,7 +30,7 @@ if TYPE_CHECKING: from collections.abc import Iterator, Sequence - from typing import TextIO + from typing import BinaryIO, TextIO from ._model import Entry, Sense from ._text import Text @@ -212,6 +212,25 @@ def _text_or_empty(text: Text | None) -> str: return str(text) if text is not None else "" +class _Utf8Sink: + """A csv sink writing UTF-8 to a byte stream it does not own. + + csv writes CRLF row terminators, and a stdout that translates newlines + doubles the CR into a blank row between every data row, so a redirected + export is not the file ``--output`` writes. Going straight to the byte + layer settles both halves of that. A ``TextIOWrapper`` around the same + buffer would too, but it owns the buffer until detached, and an error on + the way out would take stdout down with it. + """ + + def __init__(self, buffer: BinaryIO) -> None: + self._buffer = buffer + + def write(self, text: str) -> int: + """Bytes written, not characters; csv discards the count either way.""" + return self._buffer.write(text.encode("utf-8")) + + def _cmd_export(args: argparse.Namespace) -> int: from ._zip import lift_source @@ -231,22 +250,17 @@ def _cmd_export(args: argparse.Namespace) -> int: for lang in langs: header.extend([f"gloss_{lang}", f"definition_{lang}"]) - out_file: TextIO - stdout_wrapper: io.TextIOWrapper | None = None + out_file: TextIO | None = None + sink: TextIO | _Utf8Sink if args.output is not None: - out_file = args.output.open("w", encoding="utf-8", newline="") + out_file = sink = args.output.open("w", encoding="utf-8", newline="") elif isinstance(sys.stdout, io.TextIOWrapper): - # csv writes CRLF row terminators, and a stdout that translates - # newlines doubles the CR into a blank row between every data row. - # Writing through an explicit wrapper gives a redirected run the - # same bytes --output writes, instead of platform-dependent ones. - out_file = stdout_wrapper = io.TextIOWrapper( - sys.stdout.buffer, encoding="utf-8", newline="" - ) - else: # a replaced stdout need not have a byte layer to wrap - out_file = sys.stdout + sys.stdout.flush() # nothing of its own may sit behind these bytes + sink = _Utf8Sink(sys.stdout.buffer) + else: # a replaced stdout need not have a byte layer to write to + sink = sys.stdout try: - writer = csv.writer(out_file, delimiter="\t" if args.tsv else ",") + writer = csv.writer(sink, delimiter="\t" if args.tsv else ",") writer.writerow(header) with open_reader(lift_path) as reader: for entry in reader: @@ -260,10 +274,7 @@ def _cmd_export(args: argparse.Namespace) -> int: row.append(_text_or_empty(sense.definition.get(lang))) writer.writerow(row) finally: - out_file.flush() - if stdout_wrapper is not None: - stdout_wrapper.detach() # leave sys.stdout usable - elif args.output is not None: + if out_file is not None: out_file.close() return 0 @@ -276,15 +287,17 @@ def _force_utf8(stream: TextIO) -> None: one unrepresentable character killed the command mid-output. ``-o`` has always forced UTF-8; this makes a redirect agree with it. - Errors stay strict: the only content UTF-8 cannot encode is a lone - surrogate, and no file can carry one into the CLI — both readers reject it, - and the finding that reports one escapes it anyway. + Only the encoding changes. ``reconfigure`` resets the error handler to + ``strict`` unless it is passed one, and each stream's own is worth keeping: + stderr is ``backslashreplace`` so a message always arrives, and stdout is + ``surrogateescape`` on some platforms, which is what round-trips a filename + the filesystem encoding could not decode. """ if not isinstance(stream, io.TextIOWrapper): # a replaced stream may be anything return if codecs.lookup(stream.encoding).name == "utf-8": return - stream.reconfigure(encoding="utf-8") + stream.reconfigure(encoding="utf-8", errors=stream.errors) def main(argv: Sequence[str] | None = None) -> int: diff --git a/tests/test_cli.py b/tests/test_cli.py index 0722bf1..7bea9fd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -13,14 +13,18 @@ CORPUS_DIR = Path(__file__).parent / "corpus" -def _redirect(monkeypatch: pytest.MonkeyPatch, name: str, encoding: str) -> io.BytesIO: +def _redirect( + monkeypatch: pytest.MonkeyPatch, name: str, encoding: str, errors: str = "strict" +) -> io.BytesIO: """Stand in for a redirected standard stream: a byte sink under a locale codec. ``newline="\r\n"`` is what a Windows stream does with no newline argument, spelled explicitly so the CRLF-doubling regression is testable anywhere. + ``errors`` is the handler CPython hands that stream: strict for stdout + (surrogateescape on some platforms), backslashreplace for stderr. """ raw = io.BytesIO() - stream = io.TextIOWrapper(raw, encoding=encoding, newline="\r\n") + stream = io.TextIOWrapper(raw, encoding=encoding, errors=errors, newline="\r\n") monkeypatch.setattr(sys, name, stream) return raw @@ -321,7 +325,7 @@ def test_export_to_a_locale_stdout_matches_the_output_flag( def test_error_message_survives_an_ascii_stderr(monkeypatch: pytest.MonkeyPatch) -> None: - raw = _redirect(monkeypatch, "stderr", "ascii") + raw = _redirect(monkeypatch, "stderr", "ascii", errors="backslashreplace") assert main(["validate", str(CORPUS_DIR / "Órfão.lift")]) == 2 sys.stderr.flush() assert "Órfão.lift" in raw.getvalue().decode("utf-8") @@ -337,3 +341,25 @@ def test_a_stdout_that_is_not_a_text_wrapper_is_left_alone( assert main(["validate", str(CORPUS_DIR / "ranges" / "test20080407.lift")]) == 0 assert "entry_id" in sink.getvalue() assert "0 error(s), 0 warning(s)" in sink.getvalue() + + +def test_each_stream_keeps_its_own_error_handler(monkeypatch: pytest.MonkeyPatch) -> None: + """Only the encoding is forced; reconfiguring resets the handler otherwise.""" + _redirect(monkeypatch, "stdout", "cp1252", errors="surrogateescape") + _redirect(monkeypatch, "stderr", "cp1252", errors="backslashreplace") + assert main(["validate", str(CORPUS_DIR / "ranges" / "test20080407.lift")]) == 0 + assert (sys.stdout.encoding, sys.stdout.errors) == ("utf-8", "surrogateescape") + assert (sys.stderr.encoding, sys.stderr.errors) == ("utf-8", "backslashreplace") + + +def test_export_to_a_broken_pipe_leaves_stdout_usable(monkeypatch: pytest.MonkeyPatch) -> None: + """The csv sink borrows stdout's byte layer, so it must not close it.""" + + def broken(data: object) -> int: + raise BrokenPipeError(32, "Broken pipe") + + raw = _redirect(monkeypatch, "stdout", "utf-8") + monkeypatch.setattr(raw, "write", broken) + assert main(["export", str(CORPUS_DIR / "spec-examples" / "0.13" / "full-entry.lift")]) == 2 + assert not raw.closed + assert not sys.stdout.closed From 2c679b5588726e979139f7794a9c247bb127012b Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 25 Aug 2026 11:48:21 -0400 Subject: [PATCH 4/6] Say why a replaced stdout keeps its own encoding in export The fallback writes to sys.stdout as it found it, which reads as an oversight next to the two branches above it that pin UTF-8 and the row terminator. A stream with no byte layer has no encoding for this command to force -- the caller encodes it, wherever they take it -- so the choice is theirs by construction rather than by omission. Co-authored-by: Claude Opus 5 (1M context) --- src/sil_lift/_cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sil_lift/_cli.py b/src/sil_lift/_cli.py index f7d80af..b8b2533 100644 --- a/src/sil_lift/_cli.py +++ b/src/sil_lift/_cli.py @@ -257,7 +257,9 @@ def _cmd_export(args: argparse.Namespace) -> int: elif isinstance(sys.stdout, io.TextIOWrapper): sys.stdout.flush() # nothing of its own may sit behind these bytes sink = _Utf8Sink(sys.stdout.buffer) - else: # a replaced stdout need not have a byte layer to write to + else: + # A replaced stdout may have no byte layer to encode to at all, so + # its encoding and newline handling stay the caller's own choice. sink = sys.stdout try: writer = csv.writer(sink, delimiter="\t" if args.tsv else ",") From f128170fdc6bcf75b988b9b27386523718d12192 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 25 Aug 2026 12:18:22 -0400 Subject: [PATCH 5/6] Trim the csv sink's docstring to what constrains an editor The rejected TextIOWrapper is history, not a constraint, and the borrowed-stream invariant it argued for is already the class's summary line. What is left is the newline doubling, which nothing in the code hints at, and the prohibition on closing or detaching, which a later edit could undo without noticing. Co-authored-by: Claude Opus 5 (1M context) --- src/sil_lift/_cli.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/sil_lift/_cli.py b/src/sil_lift/_cli.py index b8b2533..27d937b 100644 --- a/src/sil_lift/_cli.py +++ b/src/sil_lift/_cli.py @@ -215,12 +215,11 @@ def _text_or_empty(text: Text | None) -> str: class _Utf8Sink: """A csv sink writing UTF-8 to a byte stream it does not own. - csv writes CRLF row terminators, and a stdout that translates newlines - doubles the CR into a blank row between every data row, so a redirected - export is not the file ``--output`` writes. Going straight to the byte - layer settles both halves of that. A ``TextIOWrapper`` around the same - buffer would too, but it owns the buffer until detached, and an error on - the way out would take stdout down with it. + csv writes CRLF row terminators, which a stdout that translates newlines + doubles into a blank row between every data row; the byte layer translates + nothing, so a redirected export is the file ``--output`` writes. Nothing + here closes or detaches the stream it borrowed — stdout has to stay usable + after a write fails partway. """ def __init__(self, buffer: BinaryIO) -> None: From b537507288efb4dbf538edc79139e7af498b360d Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 25 Aug 2026 12:21:08 -0400 Subject: [PATCH 6/6] Put each stream explanation where its reader needs it The redirected-stream helper explained its callers: why the newline argument is spelled out matters only to the test comparing bytes against --output, and what CPython really hands the standard streams matters only to the test asserting the handlers survive. Each moves onto that test; the helper is left describing what it builds. _force_utf8 now states the encoding hazard in the present tense -- it is what happens without the call, not a bug that once happened -- and says the handler a stream arrives with is deliberate in one clause rather than two sentences. Co-authored-by: Claude Opus 5 (1M context) --- src/sil_lift/_cli.py | 20 ++++++++++---------- tests/test_cli.py | 16 ++++++++++------ 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/sil_lift/_cli.py b/src/sil_lift/_cli.py index 27d937b..6e10f7f 100644 --- a/src/sil_lift/_cli.py +++ b/src/sil_lift/_cli.py @@ -283,16 +283,16 @@ def _cmd_export(args: argparse.Namespace) -> int: def _force_utf8(stream: TextIO) -> None: """Make one of the standard streams write UTF-8, whatever the locale is. - A stream that is not a console gets the locale encoding — cp1252 on - Windows, ASCII under a C/POSIX locale — which cannot hold LIFT content, so - one unrepresentable character killed the command mid-output. ``-o`` has - always forced UTF-8; this makes a redirect agree with it. - - Only the encoding changes. ``reconfigure`` resets the error handler to - ``strict`` unless it is passed one, and each stream's own is worth keeping: - stderr is ``backslashreplace`` so a message always arrives, and stdout is - ``surrogateescape`` on some platforms, which is what round-trips a filename - the filesystem encoding could not decode. + Off a console, a stream takes the locale encoding — cp1252 on Windows, + ASCII under a C/POSIX locale — which cannot hold LIFT content, so one + unrepresentable character kills the command mid-output. ``-o`` has always + forced UTF-8; this makes a redirect agree with it. + + Only the encoding changes: ``reconfigure`` resets the error handler to + ``strict`` unless passed one, and the handler a stream arrives with is + deliberate — ``backslashreplace`` on stderr so a message always arrives, + ``surrogateescape`` on some platforms' stdout so an undecodable filename + round-trips. """ if not isinstance(stream, io.TextIOWrapper): # a replaced stream may be anything return diff --git a/tests/test_cli.py b/tests/test_cli.py index 7bea9fd..0b6f666 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -16,12 +16,11 @@ def _redirect( monkeypatch: pytest.MonkeyPatch, name: str, encoding: str, errors: str = "strict" ) -> io.BytesIO: - """Stand in for a redirected standard stream: a byte sink under a locale codec. + """Replace a standard stream with a stand-in for a redirected one. - ``newline="\r\n"`` is what a Windows stream does with no newline argument, - spelled explicitly so the CRLF-doubling regression is testable anywhere. - ``errors`` is the handler CPython hands that stream: strict for stdout - (surrogateescape on some platforms), backslashreplace for stderr. + A text stream over the returned byte sink, under the given codec and + translating newlines the way a stream given no newline argument does on + Windows. """ raw = io.BytesIO() stream = io.TextIOWrapper(raw, encoding=encoding, errors=errors, newline="\r\n") @@ -314,6 +313,7 @@ def test_validate_text_output_survives_a_cp1252_stdout(monkeypatch: pytest.Monke def test_export_to_a_locale_stdout_matches_the_output_flag( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: + """The stand-in translates newlines, so csv's doubled CR reproduces anywhere.""" source = CORPUS_DIR / "spec-examples" / "0.13" / "multiple-forms.lift" to_file = tmp_path / "out.csv" assert main(["export", str(source), "-o", str(to_file)]) == 0 @@ -344,7 +344,11 @@ def test_a_stdout_that_is_not_a_text_wrapper_is_left_alone( def test_each_stream_keeps_its_own_error_handler(monkeypatch: pytest.MonkeyPatch) -> None: - """Only the encoding is forced; reconfiguring resets the handler otherwise.""" + """Only the encoding is forced; reconfiguring resets the handler otherwise. + + The two handlers are the ones CPython hands the real streams: + surrogateescape on stdout (on some platforms), backslashreplace on stderr. + """ _redirect(monkeypatch, "stdout", "cp1252", errors="surrogateescape") _redirect(monkeypatch, "stderr", "cp1252", errors="backslashreplace") assert main(["validate", str(CORPUS_DIR / "ranges" / "test20080407.lift")]) == 0