diff --git a/bench/README.md b/bench/README.md index 1ece45c..723ecdc 100644 --- a/bench/README.md +++ b/bench/README.md @@ -14,6 +14,7 @@ Each harness downloads what it needs into a scratch directory. | --- | --- | --- | --- | | [`icdar2013/`](icdar2013/) | ICDAR 2013 Table Competition (125 PDFs) | table detection + structure | [2026-08-02](../docs/evaluations/2026-08-02-icdar2013-table-structure.md) | | [`icdar2013/compare.py`](icdar2013/compare.py) | same, 13 systems | pdfgrab vs the whole field | [2026-09-17](../docs/evaluations/2026-09-17-field-comparison-and-metric-correction.md) | +| [`icdar2013/sweep.py`](icdar2013/sweep.py) | same, pdfgrab configs only | whether a threshold explains a gap | [2026-09-17](../docs/evaluations/2026-09-17-region-gridding-threshold-sweep.md) | | [`icdar2013/oracle.py`](icdar2013/oracle.py) | same, with ground-truth boundaries | the ceiling a layout model could reach | [2026-08-03](../docs/evaluations/2026-08-03-hybrid-ceiling-oracle-boundaries.md) | ## Why the numbers live in `docs/evaluations/` diff --git a/bench/icdar2013/extract.go b/bench/icdar2013/extract.go index f8da65c..70f16ef 100644 --- a/bench/icdar2013/extract.go +++ b/bench/icdar2013/extract.go @@ -27,6 +27,9 @@ func main() { "lines | text | mixed | auto | lines-then-mixed | fallback") merge := flag.Bool("merge", false, "TableSettings.MergeSplitTokens") detect := flag.Bool("detect", false, "TableSettings.DetectRegions (text-alignment region detection)") + minwv := flag.Int("minwv", 0, "TableSettings.MinWordsVertical (0 = leave default)") + minwh := flag.Int("minwh", 0, "TableSettings.MinWordsHorizontal (0 = leave default)") + pad := flag.Float64("pad", 0, "TextEdge.PadLines in average line heights (0 = default, negative = none)") oracle := flag.String("oracle", "", `JSON of per-page explicit edges: {"1":{"v":[..],"h":[..]}}`) flag.Parse() @@ -76,6 +79,18 @@ func main() { if *detect { for i := range attempts { attempts[i].DetectRegions = true + attempts[i].TextEdge.PadLines = *pad + } + } + + // Threshold overrides for the HAL-1363 sweep. Zero leaves whatever + // the strategy preset chose, so an unswept run is unaffected. + for i := range attempts { + if *minwv > 0 { + attempts[i].MinWordsVertical = *minwv + } + if *minwh > 0 { + attempts[i].MinWordsHorizontal = *minwh } } diff --git a/bench/icdar2013/sweep.py b/bench/icdar2013/sweep.py new file mode 100644 index 0000000..4e8ade9 --- /dev/null +++ b/bench/icdar2013/sweep.py @@ -0,0 +1,127 @@ +"""Sweep the in-region gridding thresholds (HAL-1363). + + python bench/icdar2013/sweep.py [--limit N] + +Region detection raised recall but left precision at 0.281 against +camelot's 0.514. The suspicion is configuration rather than algorithm: +MinWordsVertical/Horizontal were tuned for a whole page, and inside a +five-row region the statistics that made 3 sensible no longer hold. + +This answers that before any algorithm work. If precision moves +materially on a threshold alone, the fix is a default; if it does not, +the gridding logic itself is the problem and this saves the effort of +finding that out the slow way. + +Scores only pdfgrab configurations — the competitors do not change, and +re-running them would triple the wall-clock for no information. +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from collections import Counter + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) + +from compare import find_pairs, relations # noqa: E402 +from score import gt_relations, prf, score # noqa: E402 + + +def run_cfg(exe: str, pdf: str, cfg: dict) -> Counter: + cmd = [exe, "-strategy", cfg["strategy"]] + if cfg.get("detect"): + cmd.append("-detect") + if cfg.get("minwv"): + cmd += ["-minwv", str(cfg["minwv"])] + if cfg.get("minwh"): + cmd += ["-minwh", str(cfg["minwh"])] + if "pad" in cfg: + cmd += ["-pad", str(cfg["pad"])] + cmd.append(pdf) + try: + out = subprocess.run(cmd, capture_output=True, timeout=120).stdout + return relations([t["rows"] for t in json.loads(out or b"[]")]) + except Exception: + return Counter() + + +def configs() -> list[dict]: + out = [{"name": "baseline text+detect", "strategy": "text", "detect": True}] + + # One axis at a time, so a move can be attributed. + for n in (1, 2): + out.append({"name": f"minwv={n}", "strategy": "text", "detect": True, "minwv": n}) + out.append({"name": f"minwh={n}", "strategy": "text", "detect": True, "minwh": n}) + + # Both together, in case the effect only appears jointly. + out.append({"name": "minwv=1 minwh=1", "strategy": "text", "detect": True, + "minwv": 1, "minwh": 1}) + out.append({"name": "minwv=2 minwh=2", "strategy": "text", "detect": True, + "minwv": 2, "minwh": 2}) + + # Region padding: too much drags prose in, too little clips a row. + for pad in (-1, 0.5, 2.0, 3.0, 4.0): + label = "none" if pad < 0 else pad + out.append({"name": f"pad={label}", "strategy": "text", "detect": True, "pad": pad}) + + return out + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("root") + ap.add_argument("exe") + ap.add_argument("--limit", type=int, default=0) + ap.add_argument("--json", default="") + args = ap.parse_args() + + pairs = find_pairs(args.root, args.limit) + cfgs = configs() + print(f"documents : {len(pairs)}") + print(f"configs : {len(cfgs)}\n", flush=True) + + per_doc = {c["name"]: [] for c in cfgs} + + for i, (pdf, xml) in enumerate(pairs, 1): + gt = gt_relations(xml) + for c in cfgs: + got = run_cfg(args.exe, pdf, c) + per_doc[c["name"]].append(prf(*score(gt, got))) + if i % 10 == 0: + print(f" ...{i}/{len(pairs)}", flush=True) + + rows = [] + for c in cfgs: + d = per_doc[c["name"]] + n = len(d) or 1 + rows.append({ + "config": c["name"], + "precision": round(sum(x[0] for x in d) / n, 3), + "recall": round(sum(x[1] for x in d) / n, 3), + "f1": round(sum(x[2] for x in d) / n, 3), + }) + rows.sort(key=lambda r: r["f1"], reverse=True) + + w = max(len(r["config"]) for r in rows) + 2 + print(f"\n{'config':<{w}} {'prec':>7} {'recall':>7} {'F1':>7}") + print("-" * (w + 24)) + for r in rows: + print(f"{r['config']:<{w}} {r['precision']:>7.3f} {r['recall']:>7.3f} {r['f1']:>7.3f}") + + print("\nReference on this corpus: pdfgrab lines F1 0.442 · camelot stream") + print("P 0.514 R 0.762 F1 0.582. Per-document averaging, end-to-end.") + + if args.json: + with open(args.json, "w") as fh: + json.dump({"documents": len(pairs), "results": rows}, fh, indent=2) + print(f"\nwrote {args.json}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/detect_textedge.go b/detect_textedge.go index 0970fd9..ea5fc57 100644 --- a/detect_textedge.go +++ b/detect_textedge.go @@ -98,6 +98,18 @@ type TextEdgeOpts struct { // LineTol is the vertical tolerance for grouping words into a line. LineTol float64 + + // PadLines grows each detected region vertically by this many + // average line heights, so a header or trailing row sitting just + // outside the detected alignment is not clipped away. + // + // It trades recall for precision directly: too little and the first + // or last row is lost, too much and neighbouring prose is pulled + // into the region and gridded. + // + // Zero means the default, consistent with every other field here. + // Pass a negative value for no padding at all. + PadLines float64 } // DefaultTextEdgeOpts returns the published parameters. @@ -108,6 +120,7 @@ func DefaultTextEdgeOpts() TextEdgeOpts { CoordTol: 0.5, MinTextLen: 2, LineTol: 2, + PadLines: 1, } } @@ -207,6 +220,13 @@ func withTextEdgeDefaults(o TextEdgeOpts) TextEdgeOpts { if o.LineTol <= 0 { o.LineTol = d.LineTol } + switch { + case o.PadLines == 0: + o.PadLines = d.PadLines + case o.PadLines < 0: + // Explicitly no padding. + o.PadLines = 0 + } return o } @@ -376,7 +396,7 @@ func regionsFromEdges(edges []*textEdge, lines []textLine, opts TextEdgeOpts) [] } } - pad := averageLineHeight(lines) + pad := averageLineHeight(lines) * opts.PadLines out := make([]BBox, 0, len(bands)) for _, b := range bands { diff --git a/detect_textedge_test.go b/detect_textedge_test.go index 2391460..be260fd 100644 --- a/detect_textedge_test.go +++ b/detect_textedge_test.go @@ -240,3 +240,29 @@ func TestIsDeterministic(t *testing.T) { } } } + +// PadLines follows the same zero-means-default rule as every other +// field, with a negative value as the explicit "none". Getting this +// backwards would silently disable padding for anyone using a partially +// filled TextEdgeOpts. +func TestPadLinesZeroMeansDefault(t *testing.T) { + ws := tableWords([]float64{72, 200, 330}, 6, 700, 20) + + def := DetectTextEdgeRegions(ws, DefaultTextEdgeOpts()) + zero := DetectTextEdgeRegions(ws, TextEdgeOpts{}) + if len(def) != 1 || len(zero) != 1 { + t.Fatalf("expected one region each, got %d and %d", len(def), len(zero)) + } + if math.Abs(def[0].Y0-zero[0].Y0) > 1e-9 || math.Abs(def[0].Y1-zero[0].Y1) > 1e-9 { + t.Errorf("zero PadLines did not fall back to the default: %v vs %v", zero[0], def[0]) + } + + none := DetectTextEdgeRegions(ws, TextEdgeOpts{PadLines: -1}) + if len(none) != 1 { + t.Fatalf("expected one region with padding off, got %d", len(none)) + } + if none[0].Y1 >= def[0].Y1 || none[0].Y0 <= def[0].Y0 { + t.Errorf("negative PadLines should shrink the region: %v vs default %v", + none[0], def[0]) + } +} diff --git a/docs/README.md b/docs/README.md index 3832777..3dbd03f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -20,6 +20,7 @@ release history in [`CHANGELOG.md`](../CHANGELOG.md). | date | subject | headline | | --- | --- | --- | +| [2026-09-17](evaluations/2026-09-17-region-gridding-threshold-sweep.md) | in-region gridding threshold sweep | **negative** — 12 configs, precision never leaves 0.28. Not a tuning problem; the gridding algorithm is | | [2026-09-17](evaluations/2026-09-17-text-edge-region-detection.md) | text-edge region detection (Nurminen) | **partial** — F1 0.245 → 0.337 over page-wide `text`, but below `lines` 0.442. Recall transferred, precision did not: gridding is now the bottleneck | | [2026-09-17](evaluations/2026-09-17-field-comparison-and-metric-correction.md) | the field, and a metric correction | **per-doc F1 0.442**, 5th of 10, ~10x faster than anything comparable. No Go library comes close. Earlier figures were pooled, not the competition's metric | | [2026-08-03](evaluations/2026-08-03-hybrid-ceiling-oracle-boundaries.md) | hybrid ceiling with oracle boundaries | **0.362 → 0.935.** Given a correct grid, extraction is near-perfect — structure is the whole gap | diff --git a/docs/evaluations/2026-09-17-region-gridding-threshold-sweep.md b/docs/evaluations/2026-09-17-region-gridding-threshold-sweep.md new file mode 100644 index 0000000..5e13a4c --- /dev/null +++ b/docs/evaluations/2026-09-17-region-gridding-threshold-sweep.md @@ -0,0 +1,84 @@ +# The in-region precision gap is not a tuning problem + +**Date:** 2026-09-17 +**Harness:** [`bench/icdar2013/sweep.py`](../../bench/icdar2013/sweep.py) +**Corpus:** ICDAR 2013, Smock-corrected — 125 PDFs, 39,524 relations +**Question:** region detection left precision at 0.281 against camelot's 0.514. Is that a threshold that was tuned for a page and is wrong for a region? + +## Answer: no. Twelve configurations, and precision never leaves 0.28. + +| config | precision | recall | **F1** | +|---|---|---|---| +| `pad=none` | 0.289 | 0.629 | **0.342** | +| `pad=0.5` | 0.288 | 0.630 | **0.342** | +| `minwh=2` | 0.283 | 0.635 | **0.338** | +| **baseline (`text`+detect)** | 0.281 | 0.635 | **0.337** | +| `minwh=1` | 0.281 | 0.635 | 0.337 | +| `pad=2.0` | 0.274 | 0.642 | 0.332 | +| `pad=3.0` | 0.261 | 0.637 | 0.322 | +| `pad=4.0` | 0.247 | 0.623 | 0.308 | +| `minwv=2 minwh=2` | 0.229 | 0.606 | 0.284 | +| `minwv=2` | 0.227 | 0.606 | 0.283 | +| `minwv=1` | 0.128 | 0.464 | 0.167 | +| `minwv=1 minwh=1` | 0.128 | 0.464 | 0.167 | + +*Reference: pdfgrab `lines` 0.442 · camelot `stream` P 0.514 R 0.762 F1 0.582.* + +Best is `pad=none` at 0.342 against a baseline of 0.337. **+0.005 is noise**, not +a finding. The entire reachable precision range is 0.128–0.289, and camelot's +0.514 is nowhere in it. + +## The stated hypothesis was backwards + +HAL-1363 proposed that `MinWordsVertical` (default 3) was *too high* for a small +region: a five-row table only has five words per column, so a page-tuned +threshold should be producing spurious boundaries. + +The opposite is true. **Lowering it is catastrophic** — `minwv=1` halves +precision to 0.128 and takes recall down with it. The threshold is not too +strict; it is the only thing holding precision up at all. + +Raising it (`minwv=2` is lower than the default 3, so the default is already the +strictest tested) also loses ground, so the default sits at or near the optimum +for this axis. There is no room on this parameter. + +## Region padding behaves, and the small sample lied + +An early three-document run put `pad=2.0` on top at F1 0.628 with recall 0.838. +At 125 documents that config is **sixth**, and padding beyond the default only +degrades — 2.0 → 3.0 → 4.0 costs precision monotonically (0.274 → 0.261 → 0.247) +for no recall worth having. + +Worth recording as a method note rather than a footnote: a three-document +sample inverted the ranking of the best and sixth-best configurations. The +cheap smoke test is for checking the harness runs, never for reading a result +off. + +## What this rules out, and what it leaves + +Ruled out: **the precision gap is not a configuration difference.** No +combination of the exposed thresholds moves it, so no default change will fix +it and no further sweeping is worth the wall-clock. + +What is left is the gridding algorithm itself. Given the same detected region, +camelot divides it into cells differently, and that difference is worth roughly +**0.23 precision** — far too large to be tuning. Reading `_generate_columns_and_rows` +in `camelot/parsers/base.py` is now the only sensible next step. + +The oracle result still bounds the opportunity: handed a **correct** grid, this +same extractor reaches **0.935**. Nothing here suggests the ceiling is the +problem. + +## Cost of learning this + +Roughly ten minutes of compute, twice, against the days that tuning thresholds +by hand would have taken to reach the same conclusion less certainly. The sweep +script is committed so the next parameter question is a one-liner. + +## Reproduce + +```sh +python bench/icdar2013/sweep.py \ + ~/.cache/pdfgrab-bench/ICDAR-2013-Table-Competition-Corrected \ + ~/.cache/pdfgrab-bench/bench-extract +```