diff --git a/docs/en/guides/cli.md b/docs/en/guides/cli.md index 7953a37..bbd9443 100644 --- a/docs/en/guides/cli.md +++ b/docs/en/guides/cli.md @@ -67,4 +67,6 @@ senses: 4541 $ sil-lift export dictionary.lift --langs en,fr -o dictionary.csv ``` +All output is UTF-8, on every platform and whether it goes to a console, a pipe, or a `>` redirect — never the locale encoding (cp1252 on Windows, ASCII under a C/POSIX locale), which cannot represent LIFT content. `sil-lift export dictionary.lift > dictionary.csv` therefore writes exactly the bytes `-o dictionary.csv` writes, CRLF row terminators included. + Exit codes: `0` success (warnings allowed, unless `--strict`), `1` findings (validation errors / missing media / warnings under `--strict`), `2` unreadable input. diff --git a/src/sil_lift/_cli.py b/src/sil_lift/_cli.py index e72c083..771fc79 100644 --- a/src/sil_lift/_cli.py +++ b/src/sil_lift/_cli.py @@ -12,7 +12,9 @@ from __future__ import annotations import argparse +import codecs import csv +import io import json import sys import tempfile @@ -230,10 +232,19 @@ def _cmd_export(args: argparse.Namespace) -> int: header.extend([f"gloss_{lang}", f"definition_{lang}"]) out_file: TextIO - if args.output is None: - out_file = sys.stdout - else: + stdout_wrapper: io.TextIOWrapper | None = None + if args.output is not None: out_file = args.output.open("w", encoding="utf-8", newline="") + elif isinstance(sys.stdout, io.TextIOWrapper): + # csv writes CRLF row terminators, and a stdout that translates + # newlines doubles the CR into a blank row between every data row. + # Writing through an explicit wrapper gives a redirected run the + # same bytes --output writes, instead of platform-dependent ones. + out_file = stdout_wrapper = io.TextIOWrapper( + sys.stdout.buffer, encoding="utf-8", newline="" + ) + else: # a replaced stdout need not have a byte layer to wrap + out_file = sys.stdout try: writer = csv.writer(out_file, delimiter="\t" if args.tsv else ",") writer.writerow(header) @@ -249,12 +260,36 @@ def _cmd_export(args: argparse.Namespace) -> int: row.append(_text_or_empty(sense.definition.get(lang))) writer.writerow(row) finally: - if args.output is not None: + out_file.flush() + if stdout_wrapper is not None: + stdout_wrapper.detach() # leave sys.stdout usable + elif args.output is not None: out_file.close() return 0 +def _force_utf8(stream: TextIO) -> None: + """Make one of the standard streams write UTF-8, whatever the locale is. + + A stream that is not a console gets the locale encoding — cp1252 on + Windows, ASCII under a C/POSIX locale — which cannot hold LIFT content, so + one unrepresentable character killed the command mid-output. ``-o`` has + always forced UTF-8; this makes a redirect agree with it. + + Errors stay strict: the only content UTF-8 cannot encode is a lone + surrogate, and no file can carry one into the CLI — both readers reject it, + and the finding that reports one escapes it anyway. + """ + if not isinstance(stream, io.TextIOWrapper): # a replaced stream may be anything + return + if codecs.lookup(stream.encoding).name == "utf-8": + return + stream.reconfigure(encoding="utf-8") + + def main(argv: Sequence[str] | None = None) -> int: + _force_utf8(sys.stdout) + _force_utf8(sys.stderr) parser = argparse.ArgumentParser( prog="sil-lift", description="Utilities for LIFT 0.13 lexicon files.", diff --git a/tests/test_cli.py b/tests/test_cli.py index 28c24bb..0722bf1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,6 +2,8 @@ import io import json import shutil +import sys +import unicodedata from pathlib import Path import pytest @@ -11,6 +13,18 @@ CORPUS_DIR = Path(__file__).parent / "corpus" +def _redirect(monkeypatch: pytest.MonkeyPatch, name: str, encoding: str) -> io.BytesIO: + """Stand in for a redirected standard stream: a byte sink under a locale codec. + + ``newline="\r\n"`` is what a Windows stream does with no newline argument, + spelled explicitly so the CRLF-doubling regression is testable anywhere. + """ + raw = io.BytesIO() + stream = io.TextIOWrapper(raw, encoding=encoding, newline="\r\n") + monkeypatch.setattr(sys, name, stream) + return raw + + def test_validate_clean_file(capsys: pytest.CaptureFixture[str]) -> None: assert main(["validate", str(CORPUS_DIR / "ranges" / "test20080407.lift")]) == 0 out = capsys.readouterr().out @@ -282,3 +296,44 @@ def test_export_filename_with_space(tmp_path: Path) -> None: out = tmp_path / "out.csv" assert main(["export", str(path), "-o", str(out)]) == 0 assert out.is_file() + + +def test_validate_text_output_survives_a_cp1252_stdout(monkeypatch: pytest.MonkeyPatch) -> None: + raw = _redirect(monkeypatch, "stdout", "cp1252") + assert main(["validate", str(CORPUS_DIR / "negative" / "nfd-range-ids.lift")]) == 1 + sys.stdout.flush() + out = raw.getvalue().decode("utf-8") + assert unicodedata.normalize("NFD", "Órfão") in out # the id cp1252 cannot hold + assert out.splitlines()[-1].endswith("warning(s)") # ran to the summary, not truncated + + +def test_export_to_a_locale_stdout_matches_the_output_flag( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = CORPUS_DIR / "spec-examples" / "0.13" / "multiple-forms.lift" + to_file = tmp_path / "out.csv" + assert main(["export", str(source), "-o", str(to_file)]) == 0 + + raw = _redirect(monkeypatch, "stdout", "ascii") + assert main(["export", str(source)]) == 0 + assert raw.getvalue() == to_file.read_bytes() + assert "เอว" in raw.getvalue().decode("utf-8") # a gloss ascii cannot hold + + +def test_error_message_survives_an_ascii_stderr(monkeypatch: pytest.MonkeyPatch) -> None: + raw = _redirect(monkeypatch, "stderr", "ascii") + assert main(["validate", str(CORPUS_DIR / "Órfão.lift")]) == 2 + sys.stderr.flush() + assert "Órfão.lift" in raw.getvalue().decode("utf-8") + + +def test_a_stdout_that_is_not_a_text_wrapper_is_left_alone( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A replaced stdout has no encoding to fix and no byte layer to wrap.""" + sink = io.StringIO() + monkeypatch.setattr(sys, "stdout", sink) + assert main(["export", str(CORPUS_DIR / "spec-examples" / "0.13" / "full-entry.lift")]) == 0 + assert main(["validate", str(CORPUS_DIR / "ranges" / "test20080407.lift")]) == 0 + assert "entry_id" in sink.getvalue() + assert "0 error(s), 0 warning(s)" in sink.getvalue()