diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e53921c..6f6d1c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,7 +69,7 @@ jobs: uses: actions/checkout@v4 with: repository: regtab/jregtab - ref: v0.4.1 + ref: v0.5.0 token: ${{ secrets.JREGTAB_TOKEN }} path: jregtab - uses: actions/setup-java@v4 diff --git a/Cargo.lock b/Cargo.lock index 32a3cbf..cf56bda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -167,7 +167,7 @@ dependencies = [ [[package]] name = "pyregtab" -version = "0.4.0" +version = "0.5.0" dependencies = [ "indexmap", "pyo3", diff --git a/Cargo.toml b/Cargo.toml index 7233ec3..632ec95 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyregtab" -version = "0.4.0" +version = "0.5.0" edition = "2021" description = "Native core of pyRegTab: RTL compiler, ATP matcher and table interpreter" license = "MIT" diff --git a/README.md b/README.md index 286d0ab..79017b9 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,8 @@ and interprets the match into a relational **recordset**: TableSyntax → RtlCompiler/TablePattern → AtpMatcher → TableInterpreter → Recordset ``` -**pyRegTab 0.4.0 ≙ jRegTab 0.4.1** (same API, same semantics, same test -corpus; jRegTab 0.4.1 changes only the Java build over 0.4.0), including the +**pyRegTab 0.5.0 ≙ jRegTab 0.5.0** (same API, same semantics, same test +corpus), including the embedded RTL DSL `pyregtab.dsl` — a port of jRegTab's `ru.icc.regtab.dsl` (added upstream in jRegTab 0.3.0). Python-side extras on top of the Java API: `AtpMatcher.match_many` (parallel batch matching), `Recordset.to_pandas()`, @@ -165,8 +165,7 @@ Rust (`pyregtab._core`, built with [PyO3](https://pyo3.rs) and smoke test against the native core alone. Differential testing against the Java reference (`tools/differential.py` + `tools/RecordsetDumpMain.java`) compares recordsets cell-by-cell on all 750 task variants — zero -mismatches against jRegTab v0.4.0 (whose Java sources are unchanged in -v0.4.1). +mismatches against jRegTab v0.5.0. ## IDE support diff --git a/conformance/README.md b/conformance/README.md index d4e8354..737b2fd 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -14,8 +14,14 @@ conformance/ ├── positive/ │ ├── .rtl — RTL source (UTF-8, no BOM, LF, trailing newline) │ └── .expected.rtl — canonical form: serialize(compile(.rtl)) -└── negative/ - └── .rtl — must be rejected with a compile error +├── negative/ +│ └── .rtl — must be rejected with a compile error +└── semantic/ + └── / — execution semantics, see below + ├── pattern.rtl + ├── input.csv + ├── expected.csv + └── options.json — optional ``` **Byte-exactness caveat:** RTL string literals may contain raw CR/CRLF bytes as @@ -39,11 +45,67 @@ Any RTL implementation must satisfy, for this corpus: 4. Every `negative/.rtl` is **rejected** with a compile error (`RtlCompileException` in Java, `RtlCompileError` in Python). Reporting the error position is recommended but not normative. +5. For every `semantic//`, matching `pattern.rtl` against `input.csv` and + interpreting the result yields a recordset equal to `expected.csv`. Byte-equality of canonical forms transitively guarantees that two implementations build -the same ATP without comparing object graphs across languages. - -In jRegTab the contract is executed by `ru.icc.regtab.conformance.RtlConformanceTest`; +the same ATP without comparing object graphs across languages. Items 1–4 stop there, +though: two implementations can agree on the canonical form of a pattern and still +*execute* it differently. Item 5 closes that gap for the behaviours where it matters. + +## Semantic cases + +A case is a directory under `semantic/` holding: + +| File | Role | +|---|---| +| `pattern.rtl` | the RTL pattern (UTF-8, no BOM, LF, trailing newline) | +| `input.csv` | the table to match — no header row, `,` delimiter, `"` quotes, UTF-8 | +| `expected.csv` | the expected recordset, same CSV dialect | +| `options.json` | optional; `attributeOrder`, `recordOrder` (`STRICT`\|`FLEXIBLE`), `expectedHasHeader` | + +Cell text is taken from the CSV **verbatim** — quoting is what makes leading and trailing +spaces significant, so `"a, b"` is one cell whose text is `a, b`, and `""` is an empty +cell. Readers must not strip surrounding whitespace. + +By default `expected.csv` has **no header row**: its columns are matched positionally +against the schema the pattern produced, in record order. This keeps attribute names +invented by the implementation out of the contract. A case whose pattern names its +attributes (via `AVP`) may set `"expectedHasHeader": true` and put those names in the +first row. + +Cases are maintained by hand, like `negative/` — the generator never writes here. +Keep each case minimal and focused on one rule, so that a failure names the rule. + +## Semantics of S_delim + +The canonical form cannot reveal this rule — both spellings below serialize +identically — so it is pinned by `semantic/` cases. The semantics of the delimited +content specification `S_delim = (δ, S_atom)` (`def:delimited-content-spec`): + +- The input text is split on every occurrence of `δ`, keeping trailing empty fields + (Java `String.split(…, -1)`, Python `str.split(δ)`). +- Each substring `sₖ ∈ Σ*` is passed to `S_atom` **verbatim**. Implementations must not + trim substrings and must not drop empty ones: `n` substrings always derive `n` items, + numbered `0..n-1`. `"a, b"` therefore yields `"a"` and `" b"`; `"a,,b"` yields + `"a"`, `""`, `"b"`. +- Whitespace removal is opt-in, expressed by the atom's string extractor `ξ`: + `(VAL=TRIM){","}` (or `=NORM`). The extractor applies to each substring separately. +- The same rules apply to a delimited specification nested in a compound one. + +Positive case `delim_raw` pins both forms syntactically; the executable checks are +`semantic/delim_raw_tokens` (token whitespace survives), +`semantic/delim_empty_tokens` (empty tokens derive items), +`semantic/delim_trim` (`=TRIM` opts into trimming) and +`semantic/compound_delim_raw` (the same rules for a delimited segment nested in a +compound specification). + +> Changed in jRegTab 0.5.0. Earlier versions trimmed each substring and silently +> dropped empty ones; patterns relying on that must add `=TRIM` to the delimited atom. + +In jRegTab items 1–4 of the contract are executed by +`ru.icc.regtab.conformance.RtlConformanceTest` and item 5 by +`ru.icc.regtab.conformance.RtlSemanticConformanceTest`; `ConformanceCorpusFreshnessTest` additionally guards the committed files against drift from the task test suite. @@ -58,15 +120,17 @@ mvn test-compile org.codehaus.mojo:exec-maven-plugin:3.5.0:java \ -Dexec.classpathScope=test ``` -Commit the result. Negative cases are maintained by hand — when adding a new error -branch to the grammar or compiler, add a case here. +Commit the result. Negative and semantic cases are maintained by hand — when adding a +new error branch to the grammar or compiler, add a `negative/` case; when changing or +clarifying how a construct *executes*, add a `semantic/` one. ## Evolving RTL Any change to the RTL language follows this order: 1. Change the grammar `RTL.g4` (the normative specification) in jRegTab. -2. Add/extend corpus cases (positive with canonical forms, negative for new error branches). +2. Add/extend corpus cases (positive with canonical forms, negative for new error + branches, semantic for new or changed execution behaviour). 3. Implement in the jRegTab compiler; CI (`conformance` job) must be green. 4. Downstream implementations update their pinned upstream commit, sync the corpus copy, and implement the change; their conformance suite must be green. diff --git a/conformance/UPSTREAM b/conformance/UPSTREAM index c75df49..04a2f76 100644 --- a/conformance/UPSTREAM +++ b/conformance/UPSTREAM @@ -1,3 +1,3 @@ -commit: 7a9b78974cc88c872bc994d6d38c69e810828ea6 -tag: v0.4.1 +commit: 035ff1a139e885e4cea85aa66a33e89a6b30f8c9 +tag: v0.5.0 path: conformance/ diff --git a/conformance/VERSION b/conformance/VERSION index 0ac4b9d..1820d49 100644 --- a/conformance/VERSION +++ b/conformance/VERSION @@ -1,2 +1,2 @@ -generated: 2026-07-07 +generated: 2026-08-26 sources: RtlTask001..150 + curated extras diff --git a/conformance/positive/delim_raw.expected.rtl b/conformance/positive/delim_raw.expected.rtl new file mode 100644 index 0000000..ee18939 --- /dev/null +++ b/conformance/positive/delim_raw.expected.rtl @@ -0,0 +1 @@ +[ [ (VAL : CL*->REC){","} ] [ (VAL = TRIM : CL*->REC){","} ] ] diff --git a/conformance/positive/delim_raw.rtl b/conformance/positive/delim_raw.rtl new file mode 100644 index 0000000..3afa3b4 --- /dev/null +++ b/conformance/positive/delim_raw.rtl @@ -0,0 +1 @@ +[ [(VAL : CL*->REC){','}] [(VAL=TRIM : CL*->REC){','}] ] diff --git a/conformance/positive/task_045.expected.rtl b/conformance/positive/task_045.expected.rtl index b172a2f..ae5e0d3 100644 --- a/conformance/positive/task_045.expected.rtl +++ b/conformance/positive/task_045.expected.rtl @@ -1 +1 @@ - [ [ !BLANK? VAL ] [ !BLANK? (VAL : (SR & C0)->REC(1)){","} ] ]+ + [ [ !BLANK? VAL ] [ !BLANK? (VAL = TRIM : (SR & C0)->REC(1)){","} ] ]+ diff --git a/conformance/positive/task_045.rtl b/conformance/positive/task_045.rtl index 0b05e34..5b22b59 100644 --- a/conformance/positive/task_045.rtl +++ b/conformance/positive/task_045.rtl @@ -1 +1 @@ -[ [!BLANK? VAL] [!BLANK? (VAL : SR&C0->REC(1)){','}] ]+ +[ [!BLANK? VAL] [!BLANK? (VAL=TRIM : SR&C0->REC(1)){','}] ]+ diff --git a/conformance/positive/task_055.expected.rtl b/conformance/positive/task_055.expected.rtl index 9280f9e..f9653e9 100644 --- a/conformance/positive/task_055.expected.rtl +++ b/conformance/positive/task_055.expected.rtl @@ -1 +1 @@ -[ [ VAL : CL*->REC "," (VAL){","} ] ]+ +[ [ VAL : CL*->REC "," (VAL = TRIM){","} ] ]+ diff --git a/conformance/positive/task_055.rtl b/conformance/positive/task_055.rtl index 3410180..d9d86b9 100644 --- a/conformance/positive/task_055.rtl +++ b/conformance/positive/task_055.rtl @@ -1 +1 @@ -[ [VAL: CL*->REC ',' (VAL){','}] ]+ +[ [VAL: CL*->REC ',' (VAL=TRIM){','}] ]+ diff --git a/conformance/semantic/compound_delim_raw/expected.csv b/conformance/semantic/compound_delim_raw/expected.csv new file mode 100644 index 0000000..dd3f1ef --- /dev/null +++ b/conformance/semantic/compound_delim_raw/expected.csv @@ -0,0 +1,2 @@ +"a1"," b1"," c1" +"a2"," b2"," c2" diff --git a/conformance/semantic/compound_delim_raw/input.csv b/conformance/semantic/compound_delim_raw/input.csv new file mode 100644 index 0000000..f9664f4 --- /dev/null +++ b/conformance/semantic/compound_delim_raw/input.csv @@ -0,0 +1,2 @@ +"a1, b1, c1" +"a2, b2, c2" diff --git a/conformance/semantic/compound_delim_raw/pattern.rtl b/conformance/semantic/compound_delim_raw/pattern.rtl new file mode 100644 index 0000000..3410180 --- /dev/null +++ b/conformance/semantic/compound_delim_raw/pattern.rtl @@ -0,0 +1 @@ +[ [VAL: CL*->REC ',' (VAL){','}] ]+ diff --git a/conformance/semantic/delim_empty_tokens/expected.csv b/conformance/semantic/delim_empty_tokens/expected.csv new file mode 100644 index 0000000..12ee391 --- /dev/null +++ b/conformance/semantic/delim_empty_tokens/expected.csv @@ -0,0 +1,6 @@ +"k1","a" +"k1","" +"k1","b" +"k2","c" +"k2","d" +"k2","" diff --git a/conformance/semantic/delim_empty_tokens/input.csv b/conformance/semantic/delim_empty_tokens/input.csv new file mode 100644 index 0000000..d4c092c --- /dev/null +++ b/conformance/semantic/delim_empty_tokens/input.csv @@ -0,0 +1,2 @@ +"k1","a,,b" +"k2","c,d," diff --git a/conformance/semantic/delim_empty_tokens/pattern.rtl b/conformance/semantic/delim_empty_tokens/pattern.rtl new file mode 100644 index 0000000..0b05e34 --- /dev/null +++ b/conformance/semantic/delim_empty_tokens/pattern.rtl @@ -0,0 +1 @@ +[ [!BLANK? VAL] [!BLANK? (VAL : SR&C0->REC(1)){','}] ]+ diff --git a/conformance/semantic/delim_raw_tokens/expected.csv b/conformance/semantic/delim_raw_tokens/expected.csv new file mode 100644 index 0000000..c7a4a65 --- /dev/null +++ b/conformance/semantic/delim_raw_tokens/expected.csv @@ -0,0 +1,4 @@ +"k1","a" +"k1"," b" +"k2","c " +"k2","d" diff --git a/conformance/semantic/delim_raw_tokens/input.csv b/conformance/semantic/delim_raw_tokens/input.csv new file mode 100644 index 0000000..bc7b265 --- /dev/null +++ b/conformance/semantic/delim_raw_tokens/input.csv @@ -0,0 +1,2 @@ +"k1","a, b" +"k2","c ,d" diff --git a/conformance/semantic/delim_raw_tokens/pattern.rtl b/conformance/semantic/delim_raw_tokens/pattern.rtl new file mode 100644 index 0000000..0b05e34 --- /dev/null +++ b/conformance/semantic/delim_raw_tokens/pattern.rtl @@ -0,0 +1 @@ +[ [!BLANK? VAL] [!BLANK? (VAL : SR&C0->REC(1)){','}] ]+ diff --git a/conformance/semantic/delim_trim/expected.csv b/conformance/semantic/delim_trim/expected.csv new file mode 100644 index 0000000..d34b1ff --- /dev/null +++ b/conformance/semantic/delim_trim/expected.csv @@ -0,0 +1,4 @@ +"k1","a" +"k1","b" +"k2","c" +"k2","d" diff --git a/conformance/semantic/delim_trim/input.csv b/conformance/semantic/delim_trim/input.csv new file mode 100644 index 0000000..bc7b265 --- /dev/null +++ b/conformance/semantic/delim_trim/input.csv @@ -0,0 +1,2 @@ +"k1","a, b" +"k2","c ,d" diff --git a/conformance/semantic/delim_trim/pattern.rtl b/conformance/semantic/delim_trim/pattern.rtl new file mode 100644 index 0000000..5b22b59 --- /dev/null +++ b/conformance/semantic/delim_trim/pattern.rtl @@ -0,0 +1 @@ +[ [!BLANK? VAL] [!BLANK? (VAL=TRIM : SR&C0->REC(1)){','}] ]+ diff --git a/docs/index.md b/docs/index.md index 2d60fc6..0d1f73a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -109,4 +109,4 @@ Requires **Python 3.10+**; binary wheels for Windows, Linux, and macOS. --- !!! note "Status" - Current release: **0.4.0** (feature parity with jRegTab 0.4.1) · License: **MIT** · [PyPI](https://pypi.org/project/pyregtab/) · [GitHub](https://github.com/regtab/pyregtab) + Current release: **0.5.0** (feature parity with jRegTab 0.5.0) · License: **MIT** · [PyPI](https://pypi.org/project/pyregtab/) · [GitHub](https://github.com/regtab/pyregtab) diff --git a/docs/model/atp.md b/docs/model/atp.md index 4d1d869..0e03e37 100644 --- a/docs/model/atp.md +++ b/docs/model/atp.md @@ -196,6 +196,12 @@ If the cell text decomposes as `s₁ · δ · s₂ · δ · … · δ · sₙ`, applied independently to each `sₖ`, deriving one item per substring. This is used, for example, when a single cell contains a comma-separated list of values. +Each `sₖ ∈ Σ*` is passed to `S_atom` **verbatim**: substrings are not trimmed, and an +empty substring — produced by adjacent, leading or trailing delimiters — derives an item +with an empty string value rather than being discarded. Hence `n` substrings always +derive exactly `n` items. Whitespace removal is the job of the atom's string extractor +`ξ` (`=TRIM`, `=NORM`), which is applied to each substring in turn. + ??? note "API mapping — DelimitedContentSpec" ```python DelimitedContentSpec.of(atomSpec, delimiter) @@ -385,7 +391,7 @@ to cell `c`: - An *atomic* spec is used directly, deriving one item from the raw cell text. - A *delimited* spec splits the cell text by its delimiter and derives one item per - substring. + substring, passing each substring on unmodified. - A *compound* spec parses the cell text according to its delimiter structure and derives items from each component substring. - A *conditional* spec evaluates its condition against `c` and applies the diff --git a/docs/rtl-reference.md b/docs/rtl-reference.md index 30da0d7..0fc404e 100644 --- a/docs/rtl-reference.md +++ b/docs/rtl-reference.md @@ -170,11 +170,11 @@ When the cell body contains **only** a condition and nothing else, `?` must be o Examples from the test suite: ```rtl -[ [!BLANK? VAL] [!BLANK? (VAL : SR&C0->REC(1)){','}] ]+ +[ [!BLANK? VAL] [!BLANK? (VAL=TRIM : SR&C0->REC(1)){','}] ]+ ``` *(Task 45 — both cells of each row are guarded as non-blank; the second is also a delimited -cell.)* +cell, whose tokens are trimmed explicitly.)* ```rtl [ [BLANK] [] ]? @@ -310,6 +310,30 @@ Splits the cell text by `"sep"` and derives one item per token. *(Task 45 — a cell like `"a,b,c"` yields three VAL items, each forming a record bound to the row key via `SR & C0`.)* +**Splitting is verbatim.** Tokens reach the atom exactly as the split produced them, the +same way an atomic or compound cell receives its raw text: + +- surrounding whitespace is **kept** — `"a, b"` yields `"a"` and `" b"`, not `"a"` and `"b"`; +- empty tokens are **kept** — `"a,,b"` yields three items, the middle one an empty string, + and a trailing separator (`"a,b,"`) likewise yields a trailing empty item. + +This matches `pandas.Series.str.split`, which makes patterns over exploded columns +expressible without post-processing. + +To trim, ask for it — add a [string extractor](#atomic--contspec) to the delimited atom. +It is applied to each token separately: + +```rtl +[(VAL=TRIM){','}] // "a, b" -> "a", "b" +[(VAL=NORM){','}] // "a, b c" -> "a", "b c" +``` + +!!! warning "Changed in 0.5.0" + + Before 0.5.0 every token was trimmed and empty tokens were silently dropped. + Patterns that relied on this need `=TRIM` (or `=NORM`) added to the delimited atom: + `(VAL){","}` → `(VAL=TRIM){","}`. Atomic and compound specifications are unaffected. + ### Compound ``` diff --git a/grammar/UPSTREAM b/grammar/UPSTREAM index 9b8d286..2207657 100644 --- a/grammar/UPSTREAM +++ b/grammar/UPSTREAM @@ -1,4 +1,4 @@ -commit: 7a9b78974cc88c872bc994d6d38c69e810828ea6 -tag: v0.4.1 +commit: 035ff1a139e885e4cea85aa66a33e89a6b30f8c9 +tag: v0.5.0 path: src/main/antlr4/ru/icc/regtab/rtl/RTL.g4 sha256: 4fffbdf3f2dcb13935b8f062b5c6c55321de08cf130a4a28fa0845f5b48d68c0 diff --git a/plans/S_DELIM_RAW_SPLIT.md b/plans/S_DELIM_RAW_SPLIT.md new file mode 100644 index 0000000..e7036ac --- /dev/null +++ b/plans/S_DELIM_RAW_SPLIT.md @@ -0,0 +1,233 @@ +# План: сырое разбиение S_delim (паритет с jRegTab 0.5.0) + +**Статус:** РЕАЛИЗОВАН (2026-08-26; результаты и отклонения от плана — в §9) +**Дата:** 2026-08-26 +**Upstream:** `d:\YandexDisk\code2\jregtab` @ v0.5.0 (`035ff1a`), коммит поведения +`adab03f` «Pass delimited tokens through verbatim (S_delim)» +**Характер:** ломающее изменение семантики исполнения; синтаксис, грамматика +и сериализатор не затрагиваются + +--- + +## 1. Контекст + +`S_delim = (δ, S_atom)` по формальной модели (`def:delimited-content-spec`) +декомпозирует текст ячейки на подстроки `sₖ ∈ Σ*` и применяет `S_atom` к каждой +**как есть**. pyRegTab нарушал это в одном месте — `split_with_spans` в +[src/matcher.rs](../src/matcher.rs): каждый токен прогонялся через `java_trim`, +а пустые молча выбрасывались. Делимитированная спецификация оказывалась +единственной, где текст не доходил до атома сырым — атомарные и compound-сегменты +уже передавали его дословно. + +Практический мотив: в `regtab-eval-on-atbench` (ATBench из Auto-Tables, VLDB 2023) +эталоны страты *explode* порождены `pandas str.split(',')`, который сохраняет +пробелы токенов (`'a, b'` → `'a'`, `' b'`) и пустые строки. Принудительный trim делал +~41 из 48 кейсов страты невыразимыми. + +Upstream закрыл это в jRegTab 0.5.0: обрезка стала opt-in через экстрактор атома +(`=TRIM` / `=NORM`), а поведение зафиксировано новой нормативной секцией +`conformance/semantic/` и пунктом 5 контракта корпуса. + +## 2. Установленные факты (разведка перед реализацией) + +- Обрезка жила ровно в одном хелпере; оба вызова — `process_delimited` + (самостоятельная делимитированная ячейка) и `process_compound`, ветка + `ContentSpec::Delimited` — шли через него. В jRegTab она была продублирована + в двух местах. +- `java_trim` ([src/util.rs](../src/util.rs)) нужен для `Extractor::Trimmed` + ([src/spec.rs](../src/spec.rs)) — остаётся; убирался только его вызов из + `split_with_spans` и импорт в `matcher.rs`. +- Экстрактор применяется **после** split, в `process_atomic`. То есть `=TRIM` уже + тогда «вставал» на каждый токен отдельно, а на делимитированном пути был + no-op. Следствие: добавление `=TRIM` в паттерны 045/055 не меняет поведение + ни до, ни после правки ядра — это безопасно делать первым шагом. +- `process_atomic` не отбрасывает пустую строку (единственный ранний выход — + `Idd::Skip`), поэтому пустой токен корректно даёт item с `s == ""`. +- `CellItem.index` семантически наблюдаем: `FilterTerm::PosExact/PosOffset/PosRange` + сравнивают его. До правки `process_delimited` отдавал «сырую» позицию из + `enumerate` (с дырами на выброшенных токенах), а `process_compound` — плотный + счётчик. После правки оба дают непрерывные `0..n-1`, расхождение между путями + исчезает само. +- **Нюанс, которого нет в jRegTab:** `split_with_spans` возвращает байтовые спаны + (provenance токена в исходном тексте ячейки). До правки спан указывал на + обрезанный токен — отсюда вычисление `lead`. При переходе на сырые токены спан + становится `(base + start, base + start + part.len())`, а `lead` уходит совсем. + В Java спанов нет вообще — там правка сводилась к удалению двух строк. +- `CellItem.span` в Rust больше нигде не читается — только копируется в Python + (`PyCellDerivedItem`). Его док-комментарий уже описывал пост-фиксовую семантику. +- `.gitattributes` уже содержал `conformance/** -text` — побайтовый синк безопасен. +- CI (`.github/workflows/ci.yml`) гоняет `cargo test --no-default-features` и + `pytest tests -q` — новые тесты подхватываются сами, workflow править не нужно. + +## 3. Ядро + +Единственная правка поведения — `split_with_spans` в `src/matcher.rs`: + +```rust +/// Parts of a literal split, verbatim, as (part position, part text, byte span +/// in the original cell text); `base` is the offset of `text` within that cell +/// text. Per `def:delimited-content-spec` parts are passed on untrimmed and empty +/// parts are kept, so `n` parts always yield indices `0..n-1`; whitespace removal +/// is opt-in via the atom's extractor (`=TRIM` / `=NORM`). +fn split_with_spans(delim: &str, text: &str, base: usize) -> Vec<(usize, String, (usize, usize))> { + let mut out = Vec::new(); + let mut start = 0usize; + for (i, part) in split_literal(delim, text).into_iter().enumerate() { + let from = base + start; + let to = from + part.len(); + start += part.len() + delim.len(); + out.push((i, part, (from, to))); + } + out +} +``` + +Плюс удаление `java_trim` из импорта `matcher.rs`. Тела `process_delimited` +и `process_compound` не менялись: первый уже использовал `i` (теперь непрерывный), +второй — свой плотный `item_index` и передаёт `base = pos`, поэтому спан +вложенного в compound токена остаётся в координатах всего текста ячейки. + +## 4. Юнит-тесты ядра + +`split_with_spans` и `process_delimited` приватны, поэтому тесты — инлайн-модулем +`#[cfg(test)] mod tests` в конце `src/matcher.rs` (первый такой в `src/`; файловый +`src/tests.rs` остаётся под корпус и e2e). Стиль как в `tests.rs`: `snake_case` +без префикса `test_`, голые `assert_eq!`, вспомогательные `part(...)` / `item(...)` +для читаемых ожиданий. + +Прямые тесты `split_with_spans`: сохранение ведущего/хвостового пробела токена; +пустой токен внутри (`"a,,b"`) с нулевой шириной спана; краевые пустые токены +(`"a,b,"` и `",a"`); непрерывность индексов `0..n-1`; сдвиг спанов по ненулевому +`base` (случай вложения в compound) с обратной проверкой `cell[from..to] == s`; +байтовая точность на многобайтовом тексте. + +Сквозные тесты через `compile` + `match_atp` на таблице 1×1, с проверкой +`(s, span, index)`: сырые items у делимитированной ячейки; item на каждый пустой +токен; `=TRIM` как opt-in обратно в обрезку (при этом спаны остаются сырыми — они +до-экстракторные, и пустые токены не воскрешаются); делимитированный сегмент +внутри compound. + +**Проверка на дискриминирующесть.** После того как тесты стали зелёными, старое +поведение (`java_trim` + отбрасывание пустых) было временно возвращено — +упали **все 10** новых тестов, 4 прежних остались зелёными. Затем правка +восстановлена. Зелёный тест, зелёный при любом поведении, бесполезен. + +## 5. Синк conformance-корпуса + +`conformance/UPSTREAM` перепинен на `035ff1a139e885e4cea85aa66a33e89a6b30f8c9` / +`v0.5.0`; `conformance/` синкнут из jregtab **побайтово** (`git show :` +в бинарном режиме, без какой-либо EOL-конверсии). `UPSTREAM` — файл, локальный +для pyRegTab, его нужно пересоздавать после синка. + +Пришло из upstream (диффстат v0.4.1 → v0.5.0 по `conformance/` — ровно 20 файлов): + +| Файл | Изменение | +|---|---| +| `README.md` | секция «Semantics of S_delim», пункт 5 контракта, «Semantic cases», layout | +| `VERSION` | `generated: 2026-08-26` | +| `positive/task_045.rtl` + `.expected.rtl` | `VAL` → `VAL=TRIM` в делимитированном атоме | +| `positive/task_055.rtl` + `.expected.rtl` | то же | +| `positive/delim_raw.rtl` + `.expected.rtl` | новый кейс: обе формы рядом | +| `semantic/{delim_raw_tokens,delim_empty_tokens,delim_trim,compound_delim_raw}/` | 12 новых файлов | + +После синка: `positive/` = 304 файла (152 пары), `negative/` = 15, `semantic/` = 4 кейса. + +**Контроль целостности:** все 333 файла посверены с upstream побайтово — 0 расхождений; +сырой CRLF в `task_099` (compound-делимитер `'\r\n'`) и табы в `task_101` целы; +среди остальных 300 файлов ни одного «whitespace-only» диффа. + +## 6. Раннер пункта 5 контракта + +`tests/test_semantic_conformance.py` — порт +`ru.icc.regtab.conformance.RtlSemanticConformanceTest`. Параметризация glob-ом по +каталогам `conformance/semantic/*`, два теста на кейс: «кейс полный» (есть +`pattern.rtl`, `input.csv`, `expected.csv`) и «исполнение даёт `expected.csv`». + +Переиспользует из `tests/task_runner.py` готовые `load_table`, `load_recordset`, +`assert_matches`, константы `STRICT`/`FLEXIBLE`/`CONFORMANCE` — они уже портированы +с `CsvTableLoader`/`CsvRecordsetLoader`/`RecordsetAssert`. Конвейер кейса тот же, +что в `run_task_variant`: `AtpMatcher.match` → `TableInterpreter` со стратегией +`RECORD_FIRST` → `pattern.transform(...)`. + +**Ловушка:** дефолты семантических кейсов НЕ совпадают с задачными. +Java-раннер задаёт `(STRICT, STRICT, expectedHasHeader=false)`, тогда как +`task_runner.load_match_options` даёт `expectedHasHeader=True`. Переиспользовать +его нельзя — в новом файле свой мини-лоадер `options.json` (три ключа, `None`/пустая +строка не переопределяют, порядковые ключи в upper-case, неизвестные ключи +игнорируются, неизвестное значение порядка → ошибка). При `expectedHasHeader=false` +эталон сопоставляется **позиционно** со схемой, которую построил паттерн, — так +автогенерируемые имена атрибутов реализации не попадают в контракт. + +Ловушка подтвердилась на практике: до пересборки нативного ядра 3 из 4 кейсов +падали именно на сырых токенах — раннер дискриминирующий. + +## 7. Паттерны, зависевшие от старого поведения + +Делимитированную спецификацию используют только задачи 045, 055, 101; ведущие +пробелы есть в 045 и 055, задача 101 на табах не затронута. + +- `tests/atp_patterns.py` — файл автогенерируемый (`tools/translate_atp.py`). + Вместо ручной правки прогнан сам генератор против jregtab@v0.5.0: он дал ровно + две изменённые строки (`pattern_045`, `pattern_055`, форма + `.extract(StringExtractor.trimmed())`) и никакого другого дрейфа. Применён его + вывод дословно. Этот прогон заодно служит проверкой, что корпус и upstream + консистентны. +- `tests/test_dsl.py` — зеркальный DSL-кейс 045 синхронизирован по образцу + upstream `DslSpikeTest.task045`: RTL-строка получает `VAL=TRIM`, DSL — + `.extract(TRIM)` перед `.split_by(",")`. + +`tests/test_rtl_tasks.py` править не потребовалось — он берёт RTL прямо из корпуса +(`task_runner.task_rtl`), поэтому починился синком. Эталоны +`tests/fixtures/tasks/**/expected_*.csv` НЕ трогались. + +## 8. Документация + +Зеркалит upstream-коммит `adab03f`: + +- `docs/model/atp.md`, «Delimited content specification» — абзац о том, что + `sₖ ∈ Σ*` передаётся в `S_atom` дословно, пустая подстрока даёт item с пустым + значением, `n` подстрок → ровно `n` items, обрезка — работа экстрактора `ξ`; + плюс уточнение формулировки фазы 1 разрешения содержимого. +- `docs/rtl-reference.md`, «### Delimited» — блок «Splitting is verbatim» + (пробелы и пустые токены сохраняются, соответствие `pandas.Series.str.split`), + рецепт `(VAL=TRIM){','}` / `(VAL=NORM){','}`, warning «Changed in 0.5.0» + с миграционной инструкцией; пример Task 45 обновлён под новую форму паттерна. + +`CHANGELOG.md` в проекте нет — не заводился. + +## 9. Результат + +``` +cargo test --no-default-features 14 passed (было 4) +pytest tests -q 1918 passed (было 1908; +8 semantic, +2 delim_raw) +tests/fixtures/ не тронуты — 0 изменённых expected_*.csv +``` + +В диффе нет других поведенческих изменений, кроме `split_with_spans`. + +**Отклонения от плана (оба несущественные):** + +1. В ожидании одного юнит-теста (`=TRIM` на `"a, ,b"`) спаны были посчитаны + неверно при написании плана — строка длиной 5, спаны `(2,3)` и `(4,5)`. + Исправлено по факту прогона; поведение ядра корректно. +2. `atp_patterns.py` планировалось править вручную в стиле локального алиаса + `TRIM = StringExtractor.trimmed()`; фактически применён дословный вывод + генератора, который использует fluent-форму `.extract(StringExtractor.trimmed())`. + +**Окружение:** в `.venv` отсутствовал `maturin` — без него pytest гонял старое +скомпилированное ядро. Установлен (`maturin 1.15.0`), пересборка +`python -m maturin develop --release`. CI ставит его сам (`pip install maturin pytest`). + +## 10. Версия + +Ломающее изменение поведения → следующий релиз минимум **0.5.0**. Бампу подлежат +`Cargo.toml`, `Cargo.lock`, `pyproject.toml` и `python/pyregtab/__init__.py` +(`__version__`). Учесть, что `docs/rtl-reference.md` уже содержит +`!!! warning "Changed in 0.5.0"`, поэтому бамп и релиз делаются вместе. + +## 11. Что осознанно не трогалось + +Грамматика `grammar/RTL.g4`, парсер, ATP→RTL сериализатор, `java_trim`, +`split_literal`, `Extractor`, эталоны `tests/fixtures/tasks/**/expected_*.csv`, +CI-workflow. Канонические формы изменились только у `task_045`/`task_055` — +из-за добавленного `=TRIM` в самих паттернах, и пришли готовыми из корпуса. diff --git a/pyproject.toml b/pyproject.toml index a16629d..1e28be1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "pyregtab" -version = "0.4.0" +version = "0.5.0" description = "pyRegTab: pattern-based extraction of recordsets from tables (RTL / ATP / ITM)" readme = "README.md" license = { text = "MIT" } diff --git a/python/pyregtab/__init__.py b/python/pyregtab/__init__.py index f427c7e..a5834a4 100644 --- a/python/pyregtab/__init__.py +++ b/python/pyregtab/__init__.py @@ -73,7 +73,7 @@ from pyregtab import dsl -__version__ = "0.4.0" +__version__ = "0.5.0" __all__ = [ "TableSyntax", "Cell", "Row", "Subrow", "Subtable", "GridPosition", diff --git a/src/matcher.rs b/src/matcher.rs index abc9984..ce145da 100644 --- a/src/matcher.rs +++ b/src/matcher.rs @@ -4,7 +4,7 @@ use crate::semantics::{ActionInst, CellItem, CtxItem, ItemId, OpInst, ProviderInst, SemanticsCore}; use crate::spec::*; use crate::syntax::SyntaxCore; -use crate::util::{java_trim, split_literal, CoreErr, CoreResult}; +use crate::util::{split_literal, CoreErr, CoreResult}; use std::sync::Arc; // ---------------------------------------------------------------- match state @@ -386,20 +386,19 @@ fn process_atomic( Ok(()) } -/// Non-empty `java_trim`med parts of a literal split as -/// (original part position, trimmed part, byte span in the original cell -/// text); `base` is the offset of `text` within that cell text. +/// Parts of a literal split, verbatim, as (part position, part text, byte span +/// in the original cell text); `base` is the offset of `text` within that cell +/// text. Per `def:delimited-content-spec` parts are passed on untrimmed and empty +/// parts are kept, so `n` parts always yield indices `0..n-1`; whitespace removal +/// is opt-in via the atom's extractor (`=TRIM` / `=NORM`). fn split_with_spans(delim: &str, text: &str, base: usize) -> Vec<(usize, String, (usize, usize))> { let mut out = Vec::new(); let mut start = 0usize; for (i, part) in split_literal(delim, text).into_iter().enumerate() { - let trimmed = java_trim(&part); - if !trimmed.is_empty() { - let lead = part.len() - part.trim_start_matches(|c: char| c <= ' ').len(); - let from = base + start + lead; - out.push((i, trimmed.to_string(), (from, from + trimmed.len()))); - } + let from = base + start; + let to = from + part.len(); start += part.len() + delim.len(); + out.push((i, part, (from, to))); } out } @@ -641,3 +640,171 @@ pub fn match_atp( Err(SemErr::Other(e)) => Err(e), } } + +// ---------------------------------------------------------------- unit tests + +/// `S_delim` (`def:delimited-content-spec`) decomposes the text into substrings +/// `s_k` in `Sigma*` and applies `S_atom` to each one verbatim: no trimming, no +/// dropping of empty parts. Whitespace removal is opt-in via the atom's string +/// extractor. Pinned normatively by `conformance/semantic/`. +#[cfg(test)] +mod tests { + use super::*; + use crate::rtl::{compile, BindingsCore}; + + // -------------------------------------------------------- split_with_spans + + /// One expected `split_with_spans` triple. + fn part(index: usize, s: &str, from: usize, to: usize) -> (usize, String, (usize, usize)) { + (index, s.to_string(), (from, to)) + } + + #[test] + fn split_keeps_token_whitespace() { + // "a, b" -> "a", " b": the leading space of the second token survives, + // and its span covers the raw token, not a trimmed one. + assert_eq!( + split_with_spans(",", "a, b", 0), + vec![part(0, "a", 0, 1), part(1, " b", 2, 4)] + ); + // Trailing whitespace survives just the same. + assert_eq!( + split_with_spans(",", "c ,d", 0), + vec![part(0, "c ", 0, 2), part(1, "d", 3, 4)] + ); + } + + #[test] + fn split_keeps_empty_tokens() { + // "a,,b" -> three parts; the middle one is empty with a zero-width span. + assert_eq!( + split_with_spans(",", "a,,b", 0), + vec![part(0, "a", 0, 1), part(1, "", 2, 2), part(2, "b", 3, 4)] + ); + } + + #[test] + fn split_keeps_edge_empty_tokens() { + // A trailing delimiter yields a trailing empty part (Java `split(_, -1)`). + assert_eq!( + split_with_spans(",", "a,b,", 0), + vec![part(0, "a", 0, 1), part(1, "b", 2, 3), part(2, "", 4, 4)] + ); + // Symmetrically for a leading one. + assert_eq!( + split_with_spans(",", ",a", 0), + vec![part(0, "", 0, 0), part(1, "a", 1, 2)] + ); + } + + #[test] + fn split_indices_are_contiguous() { + // n parts always derive n items numbered 0..n-1, whatever they contain: + // blank and empty parts no longer punch holes in the numbering. + let parts = split_with_spans(",", " ,a,, ,b, ", 0); + assert_eq!(parts.len(), 6); + assert_eq!( + parts.iter().map(|(i, _, _)| *i).collect::>(), + vec![0, 1, 2, 3, 4, 5] + ); + } + + #[test] + fn split_spans_are_shifted_by_base() { + // The compound path passes `base = pos`, so spans stay in whole-cell + // coordinates. Here " b1, c1" sits at offset 3 of "a1, b1, c1". + let cell = "a1, b1, c1"; + let parts = split_with_spans(",", &cell[3..], 3); + assert_eq!(parts, vec![part(0, " b1", 3, 6), part(1, " c1", 7, 10)]); + // Every span indexes back into the original cell text. + for (_, s, (from, to)) in parts { + assert_eq!(cell[from..to], s); + } + } + + #[test] + fn split_spans_are_byte_exact_on_multibyte_text() { + // "\u{43f}\u{440}, \u{431}" — two bytes per Cyrillic letter. + let cell = "\u{43f}\u{440}, \u{431}"; + let parts = split_with_spans(",", cell, 0); + assert_eq!( + parts, + vec![part(0, "\u{43f}\u{440}", 0, 4), part(1, " \u{431}", 5, 8)] + ); + for (_, s, (from, to)) in parts { + assert_eq!(cell[from..to], s); + } + } + + // ------------------------------------------------------- end to end (ATP) + + /// Items derived from the single cell of a 1x1 table, as (s, span, index). + fn items_of(rtl: &str, text: &str) -> Vec<(String, (usize, usize), usize)> { + let mut syntax = SyntaxCore::new(1, 1).unwrap(); + syntax.cell_mut(0, 0).set_text(text.to_string()); + let pattern = compile(rtl, &BindingsCore::default()).expect("compile"); + let sem = match_atp(&pattern, &mut syntax, Vec::new()) + .expect("match") + .expect("pattern must match"); + sem.cell_items + .iter() + .map(|it| (it.s.clone(), it.span, it.index)) + .collect() + } + + /// One expected `items_of` triple. + fn item(s: &str, from: usize, to: usize, index: usize) -> (String, (usize, usize), usize) { + (s.to_string(), (from, to), index) + } + + #[test] + fn delimited_cell_derives_raw_items() { + assert_eq!( + items_of("[ [(VAL : CL*->REC){','}] ]", "a, b"), + vec![item("a", 0, 1, 0), item(" b", 2, 4, 1)] + ); + } + + #[test] + fn delimited_cell_derives_an_item_per_empty_token() { + assert_eq!( + items_of("[ [(VAL : CL*->REC){','}] ]", "a,,b"), + vec![item("a", 0, 1, 0), item("", 2, 2, 1), item("b", 3, 4, 2)] + ); + } + + #[test] + fn trim_extractor_opts_back_into_trimming() { + // `=TRIM` applies to each substring separately, restoring the old values. + // Spans stay raw: `CellItem.span` is deliberately pre-extractor. + assert_eq!( + items_of("[ [(VAL=TRIM : CL*->REC){','}] ]", "a, b"), + vec![item("a", 0, 1, 0), item("b", 2, 4, 1)] + ); + // …but it does not resurrect the dropping of empty tokens. + assert_eq!( + items_of("[ [(VAL=TRIM : CL*->REC){','}] ]", "a, ,b"), + vec![item("a", 0, 1, 0), item("", 2, 3, 1), item("b", 4, 5, 2)] + ); + } + + #[test] + fn delimited_nested_in_compound_derives_raw_items() { + // Segment 1 is atomic ("a1"), the remainder is a delimited segment. The + // compound path numbers items with its own running counter, which stays + // contiguous, and spans stay in whole-cell coordinates. + let cell = "a1, b1, c1"; + let items = items_of("[ [VAL: CL*->REC ',' (VAL){','}] ]", cell); + assert_eq!( + items, + vec![ + item("a1", 0, 2, 0), + item(" b1", 3, 6, 1), + item(" c1", 7, 10, 2), + ] + ); + for (s, (from, to), _) in items { + assert_eq!(cell[from..to], s); + } + } +} diff --git a/tests/atp_patterns.py b/tests/atp_patterns.py index 03435ef..4048034 100644 --- a/tests/atp_patterns.py +++ b/tests/atp_patterns.py @@ -260,7 +260,7 @@ def pattern_044(): def pattern_045(): NOT_BLANK = CellMatchCondition(CellPredicate.not_blank()) SAME_SUBROW_COL0 = ItemFilterConditionSpec.and_(FilterTerm.same_subrow(), FilterTerm.col_exact(0)) - return TablePattern.of(SubtablePattern.of(Quantifier.one(), RowPattern.of(Quantifier.one_or_more(), CellPattern.of(NOT_BLANK, Quantifier.one(), AtomicContentSpec.val()), CellPattern.of(NOT_BLANK, Quantifier.one(), DelimitedContentSpec(",", AtomicContentSpec.val(ActionSpec.rec(ProviderSpec.val(SAME_SUBROW_COL0, 1), anchor_pos=1))))))) + return TablePattern.of(SubtablePattern.of(Quantifier.one(), RowPattern.of(Quantifier.one_or_more(), CellPattern.of(NOT_BLANK, Quantifier.one(), AtomicContentSpec.val()), CellPattern.of(NOT_BLANK, Quantifier.one(), DelimitedContentSpec(",", AtomicContentSpec.val(ActionSpec.rec(ProviderSpec.val(SAME_SUBROW_COL0, 1), anchor_pos=1)).extract(StringExtractor.trimmed())))))) def pattern_046(): NOT_BLANK = CellMatchCondition(CellPredicate.not_blank()) @@ -324,7 +324,7 @@ def pattern_054(): def pattern_055(): SAME_CELL = ItemFilterConditionSpec.same_cell() - return TablePattern.of(SubtablePattern.of(Quantifier.one(), RowPattern.of(Quantifier.one_or_more(), CellPattern.of(CompoundContentSpec([("", AtomicContentSpec.val(ActionSpec.rec(ProviderSpec.val(SAME_CELL, UNBOUNDED)))), (",", DelimitedContentSpec(",", AtomicContentSpec.val()))], ""))))) + return TablePattern.of(SubtablePattern.of(Quantifier.one(), RowPattern.of(Quantifier.one_or_more(), CellPattern.of(CompoundContentSpec([("", AtomicContentSpec.val(ActionSpec.rec(ProviderSpec.val(SAME_CELL, UNBOUNDED)))), (",", DelimitedContentSpec(",", AtomicContentSpec.val().extract(StringExtractor.trimmed())))], ""))))) def pattern_056(): BELOW = ItemFilterConditionSpec.below() diff --git a/tests/test_dsl.py b/tests/test_dsl.py index c3c75a8..a21d7cb 100644 --- a/tests/test_dsl.py +++ b/tests/test_dsl.py @@ -221,13 +221,13 @@ def test_task029(): def test_task045(): assert_mirrors( r""" - [ [!BLANK? VAL] [!BLANK? (VAL : SR&C0->REC(1)){','}] ]+ + [ [!BLANK? VAL] [!BLANK? (VAL=TRIM : SR&C0->REC(1)){','}] ]+ """, table( subtable( row( cell(not_blank(), VAL), - cell(not_blank(), val(rec(1, SR.and_(C(0)))).split_by(",")), + cell(not_blank(), val(rec(1, SR.and_(C(0)))).extract(TRIM).split_by(",")), ).one_or_more() ) ), diff --git a/tests/test_semantic_conformance.py b/tests/test_semantic_conformance.py new file mode 100644 index 0000000..d20c708 --- /dev/null +++ b/tests/test_semantic_conformance.py @@ -0,0 +1,93 @@ +"""RTL conformance corpus, item 5 of the contract in conformance/README.md +(port of RtlSemanticConformanceTest): + + for every semantic//, matching pattern.rtl against input.csv and + interpreting the result yields a recordset equal to expected.csv. + +Items 1-4 pin syntax and canonical form only (test_conformance.py); two +implementations can agree on the canonical RTL of a pattern and still execute +it differently. This closes that gap. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from task_runner import CONFORMANCE, FLEXIBLE, STRICT, assert_matches, load_recordset, load_table + +from pyregtab import AtpMatcher, RtlCompiler, SchemaConstructionStrategy, TableInterpreter + +SEMANTIC = CONFORMANCE / "semantic" + +PATTERN = "pattern.rtl" +INPUT = "input.csv" +EXPECTED = "expected.csv" +OPTIONS = "options.json" + +# Semantic cases default to a header-less expected.csv compared positionally +# against the schema the pattern produced, so that attribute names invented by +# the implementation never leak into the contract. A case whose pattern names +# its attributes (via AVP) can set expectedHasHeader in options.json. +# +# Note this differs from the task defaults in task_runner.load_match_options, +# where expectedHasHeader is True -- do not reuse that loader here. +DEFAULTS = {"attributeOrder": STRICT, "recordOrder": STRICT, "expectedHasHeader": False} + +CASES = sorted(p for p in SEMANTIC.iterdir() if p.is_dir()) + + +def _policy(raw, fallback: str) -> str: + if raw is None or not str(raw).strip(): + return fallback + value = str(raw).strip().upper() + if value not in (STRICT, FLEXIBLE): + raise ValueError(f"Unknown order policy: {raw} (use {STRICT} or {FLEXIBLE})") + return value + + +def load_case_options(case: Path) -> dict: + """Merged options for one semantic case; unknown keys are ignored.""" + opts = dict(DEFAULTS) + file = case / OPTIONS + if not file.is_file(): + return opts + patch = json.loads(file.read_text(encoding="utf-8-sig")) or {} + opts["attributeOrder"] = _policy(patch.get("attributeOrder"), opts["attributeOrder"]) + opts["recordOrder"] = _policy(patch.get("recordOrder"), opts["recordOrder"]) + if patch.get("expectedHasHeader") is not None: + opts["expectedHasHeader"] = bool(patch["expectedHasHeader"]) + return opts + + +@pytest.mark.parametrize("case", CASES, ids=lambda p: p.name) +def test_every_case_is_complete(case): + for required in (PATTERN, INPUT, EXPECTED): + assert (case / required).is_file(), f"missing {required} in {case}" + + +@pytest.mark.parametrize("case", CASES, ids=lambda p: p.name) +def test_semantic_case(case): + syntax = load_table(case / INPUT) + # binary read: string literals may carry raw CR/CRLF payload + pattern = RtlCompiler.compile((case / PATTERN).read_bytes().decode("utf-8")) + + itm = AtpMatcher.match(pattern, syntax) + assert itm is not None, f"pattern did not match {case.name}/{INPUT}" + + actual = pattern.transform( + TableInterpreter() + .with_strategy(SchemaConstructionStrategy.RECORD_FIRST) + .interpret(itm) + ) + + opts = load_case_options(case) + expected_path = case / EXPECTED + if opts["expectedHasHeader"]: + expected = load_recordset(expected_path) + else: + # header-less: columns matched positionally against the pattern's schema + expected = load_recordset(expected_path, actual.schema) + + assert_matches(actual, expected, opts)