Write CLI output as UTF-8 regardless of the locale encoding

docs/en/guides/cli.md

@@ -67,4 +67,6 @@ senses:    4541
 $ sil-lift export dictionary.lift --langs en,fr -o dictionary.csv
 ```
 
+All output is UTF-8, on every platform and whether it goes to a console, a pipe, or a `>` redirect — never the locale encoding (cp1252 on Windows, ASCII under a C/POSIX locale), which cannot represent LIFT content. `sil-lift export dictionary.lift > dictionary.csv` therefore writes exactly the bytes `-o dictionary.csv` writes, CRLF row terminators included.
+
 Exit codes: `0` success (warnings allowed, unless `--strict`), `1` findings (validation errors / missing media / warnings under `--strict`), `2` unreadable input.

src/sil_lift/_cli.py

@@ -12,7 +12,9 @@
 from __future__ import annotations
 
 import argparse
+import codecs
 import csv
+import io
 import json
 import sys
 import tempfile
@@ -230,10 +232,19 @@ def _cmd_export(args: argparse.Namespace) -> int:
             header.extend([f"gloss_{lang}", f"definition_{lang}"])
 
         out_file: TextIO
-        if args.output is None:
-            out_file = sys.stdout
-        else:
+        stdout_wrapper: io.TextIOWrapper | None = None
+        if args.output is not None:
             out_file = args.output.open("w", encoding="utf-8", newline="")
+        elif isinstance(sys.stdout, io.TextIOWrapper):
+            # csv writes CRLF row terminators, and a stdout that translates
+            # newlines doubles the CR into a blank row between every data row.
+            # Writing through an explicit wrapper gives a redirected run the
+            # same bytes --output writes, instead of platform-dependent ones.
+            out_file = stdout_wrapper = io.TextIOWrapper(
+                sys.stdout.buffer, encoding="utf-8", newline=""
+            )
+        else:  # a replaced stdout need not have a byte layer to wrap
+            out_file = sys.stdout
         try:
             writer = csv.writer(out_file, delimiter="\t" if args.tsv else ",")
             writer.writerow(header)
@@ -249,12 +260,36 @@ def _cmd_export(args: argparse.Namespace) -> int:
                             row.append(_text_or_empty(sense.definition.get(lang)))
                         writer.writerow(row)
         finally:
-            if args.output is not None:
+            out_file.flush()
+            if stdout_wrapper is not None:
+                stdout_wrapper.detach()  # leave sys.stdout usable
+            elif args.output is not None:
                 out_file.close()
     return 0
 
 
+def _force_utf8(stream: TextIO) -> None:
+    """Make one of the standard streams write UTF-8, whatever the locale is.
+
+    A stream that is not a console gets the locale encoding — cp1252 on
+    Windows, ASCII under a C/POSIX locale — which cannot hold LIFT content, so
+    one unrepresentable character killed the command mid-output. ``-o`` has
+    always forced UTF-8; this makes a redirect agree with it.
+
+    Errors stay strict: the only content UTF-8 cannot encode is a lone
+    surrogate, and no file can carry one into the CLI — both readers reject it,
+    and the finding that reports one escapes it anyway.
+    """
+    if not isinstance(stream, io.TextIOWrapper):  # a replaced stream may be anything
+        return
+    if codecs.lookup(stream.encoding).name == "utf-8":
+        return
+    stream.reconfigure(encoding="utf-8")
+
+
 def main(argv: Sequence[str] | None = None) -> int:
+    _force_utf8(sys.stdout)
+    _force_utf8(sys.stderr)
     parser = argparse.ArgumentParser(
         prog="sil-lift",
         description="Utilities for LIFT 0.13 lexicon files.",

tests/test_cli.py

@@ -2,6 +2,8 @@
 import io
 import json
 import shutil
+import sys
+import unicodedata
 from pathlib import Path
 
 import pytest
@@ -11,6 +13,18 @@
 CORPUS_DIR = Path(__file__).parent / "corpus"
 
 
+def _redirect(monkeypatch: pytest.MonkeyPatch, name: str, encoding: str) -> io.BytesIO:
+    """Stand in for a redirected standard stream: a byte sink under a locale codec.
+
+    ``newline="\r\n"`` is what a Windows stream does with no newline argument,
+    spelled explicitly so the CRLF-doubling regression is testable anywhere.
+    """
+    raw = io.BytesIO()
+    stream = io.TextIOWrapper(raw, encoding=encoding, newline="\r\n")
+    monkeypatch.setattr(sys, name, stream)
+    return raw
+
+
 def test_validate_clean_file(capsys: pytest.CaptureFixture[str]) -> None:
     assert main(["validate", str(CORPUS_DIR / "ranges" / "test20080407.lift")]) == 0
     out = capsys.readouterr().out
@@ -282,3 +296,44 @@ def test_export_filename_with_space(tmp_path: Path) -> None:
     out = tmp_path / "out.csv"
     assert main(["export", str(path), "-o", str(out)]) == 0
     assert out.is_file()
+
+
+def test_validate_text_output_survives_a_cp1252_stdout(monkeypatch: pytest.MonkeyPatch) -> None:
+    raw = _redirect(monkeypatch, "stdout", "cp1252")
+    assert main(["validate", str(CORPUS_DIR / "negative" / "nfd-range-ids.lift")]) == 1
+    sys.stdout.flush()
+    out = raw.getvalue().decode("utf-8")
+    assert unicodedata.normalize("NFD", "Órfão") in out  # the id cp1252 cannot hold
+    assert out.splitlines()[-1].endswith("warning(s)")  # ran to the summary, not truncated
+
+
+def test_export_to_a_locale_stdout_matches_the_output_flag(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    source = CORPUS_DIR / "spec-examples" / "0.13" / "multiple-forms.lift"
+    to_file = tmp_path / "out.csv"
+    assert main(["export", str(source), "-o", str(to_file)]) == 0
+
+    raw = _redirect(monkeypatch, "stdout", "ascii")
+    assert main(["export", str(source)]) == 0
+    assert raw.getvalue() == to_file.read_bytes()
+    assert "เอว" in raw.getvalue().decode("utf-8")  # a gloss ascii cannot hold
+
+
+def test_error_message_survives_an_ascii_stderr(monkeypatch: pytest.MonkeyPatch) -> None:
+    raw = _redirect(monkeypatch, "stderr", "ascii")
+    assert main(["validate", str(CORPUS_DIR / "Órfão.lift")]) == 2
+    sys.stderr.flush()
+    assert "Órfão.lift" in raw.getvalue().decode("utf-8")
+
+
+def test_a_stdout_that_is_not_a_text_wrapper_is_left_alone(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A replaced stdout has no encoding to fix and no byte layer to wrap."""
+    sink = io.StringIO()
+    monkeypatch.setattr(sys, "stdout", sink)
+    assert main(["export", str(CORPUS_DIR / "spec-examples" / "0.13" / "full-entry.lift")]) == 0
+    assert main(["validate", str(CORPUS_DIR / "ranges" / "test20080407.lift")]) == 0
+    assert "entry_id" in sink.getvalue()
+    assert "0 error(s), 0 warning(s)" in sink.getvalue()