From 0260f70c83e6513eef3f704b4a0a0f9264014e7a Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 11:59:09 -0600 Subject: [PATCH 01/65] cleanup: dedup parser escape handling (\u/\U gate, UTF-8 decode, \c, shorthand classes) Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/parser.mojo | 203 +++++++++++++++-------------------------- 1 file changed, 74 insertions(+), 129 deletions(-) diff --git a/emberregex/parser.mojo b/emberregex/parser.mojo index d5ff20e..83c6a36 100644 --- a/emberregex/parser.mojo +++ b/emberregex/parser.mojo @@ -409,6 +409,35 @@ struct Parser[origin: Origin](Movable): at, ) + def _utf8_tail(mut self, lead: Byte) -> UInt32: + """Codepoint of the UTF-8 sequence whose lead byte (>= 0xC0) was + just consumed: reads its 1-3 continuation bytes, stopping early at + the end of the pattern.""" + var extra = 1 + var cp = UInt32(lead) & 0x1F + if lead >= 0xF0: + extra = 3 + cp = UInt32(lead) & 0x07 + elif lead >= 0xE0: + extra = 2 + cp = UInt32(lead) & 0x0F + for _ in range(extra): + if self._at_end(): + break + cp = (cp << 6) | (UInt32(self._advance()) & 0x3F) + return cp + + def _parse_control(mut self) raises -> UInt32: + """`\\cX`, after the `c`. PCRE/Perl formula: uppercase X, then XOR + 0x40 (identical to & 0x1F for letters, differs for punctuation: + \\c{ is ';', \\c; is '{').""" + if self._at_end(): + raise _regex_error("Expected character after \\c", self.pos - 1) + var ctrl = Int(self._advance()) + if ctrl >= 97 and ctrl <= 122: + ctrl -= 32 + return UInt32(ctrl ^ 0x40) + def _skip_verbose(mut self): """Skip whitespace and # comments when verbose mode is active.""" if not self.inline_flags.verbose(): @@ -473,20 +502,7 @@ struct Parser[origin: Origin](Movable): # to the same sequence but makes `α+` quantify only the last # byte — the reason this branch exists. if self.inline_flags.unicode() and ch >= 0xC0: - var extra = 1 - var cp = UInt32(ch) & 0x1F - if ch >= 0xF0: - extra = 3 - cp = UInt32(ch) & 0x07 - elif ch >= 0xE0: - extra = 2 - cp = UInt32(ch) & 0x0F - for _ in range(extra): - if self._at_end(): - break - var cont = self._advance() - cp = (cp << 6) | (UInt32(cont) & 0x3F) - return self.ast.add_node(ASTNode.literal(cp)) + return self.ast.add_node(ASTNode.literal(self._utf8_tail(ch))) return self.ast.add_node(ASTNode.literal(UInt32(ch))) def _consume_verbs(mut self) raises: @@ -751,32 +767,45 @@ struct Parser[origin: Origin](Movable): elif ch == CHAR_Z_LOWER or ch == CHAR_Z_UPPER: return self.ast.add_node(ASTNode.anchor(AnchorKind.EOS)) - # Horizontal / vertical whitespace classes, PCRE and Hyperscan - # semantics. NOTE the deliberate divergence from Python, where \v - # is the single vertical-tab character rather than a class - # (decided 2026-07-27; Hyperscan parity is this plan's goal and - # \v previously errored, so nothing silently changed meaning). - if ch == CHAR_H_LOWER or ch == CHAR_H_UPPER: - var cs = CharSet() - cs.add_range(UInt32(CHAR_SPACE), UInt32(CHAR_SPACE)) - cs.add_range(UInt32(CHAR_TAB), UInt32(CHAR_TAB)) - if ch == CHAR_H_UPPER: - cs.negate() - cs.build_bitmap() - var cs_idx = self.ast.add_charset(cs^) - return self.ast.add_node( - ASTNode.char_class(cs_idx, ch == CHAR_H_UPPER) - ) - if ch == CHAR_V_LOWER or ch == CHAR_V_UPPER: - var cs = CharSet() - cs.add_range(UInt32(CHAR_NEWLINE), UInt32(CHAR_CR)) # \n \v \f \r - if ch == CHAR_V_UPPER: + # Shorthand classes \h \v \d \w \s; the uppercase forms (\H \V \D + # \W \S) are their complements. \h / \v are the horizontal / + # vertical whitespace classes, PCRE and Hyperscan semantics. NOTE + # the deliberate divergence from Python, where \v is the single + # vertical-tab character rather than a class (decided 2026-07-27; + # Hyperscan parity is this plan's goal and \v previously errored, + # so nothing silently changed meaning). + if ( + ch == CHAR_H_LOWER + or ch == CHAR_H_UPPER + or ch == CHAR_V_LOWER + or ch == CHAR_V_UPPER + or ch == CHAR_D_LOWER + or ch == CHAR_D_UPPER + or ch == CHAR_W_LOWER + or ch == CHAR_W_UPPER + or ch == CHAR_S_LOWER + or ch == CHAR_S + ): + var cs: CharSet + if ch == CHAR_H_LOWER or ch == CHAR_H_UPPER: + cs = CharSet() + cs.add_range(UInt32(CHAR_SPACE), UInt32(CHAR_SPACE)) + cs.add_range(UInt32(CHAR_TAB), UInt32(CHAR_TAB)) + elif ch == CHAR_V_LOWER or ch == CHAR_V_UPPER: + cs = CharSet() + cs.add_range(UInt32(CHAR_NEWLINE), UInt32(CHAR_CR)) # \n-\r + elif ch == CHAR_D_LOWER or ch == CHAR_D_UPPER: + cs = CharSet.digit() + elif ch == CHAR_W_LOWER or ch == CHAR_W_UPPER: + cs = CharSet.word() + else: + cs = CharSet.whitespace() + var negated = ch <= CHAR_Z_UPPER # the uppercase forms + if negated: cs.negate() cs.build_bitmap() var cs_idx = self.ast.add_charset(cs^) - return self.ast.add_node( - ASTNode.char_class(cs_idx, ch == CHAR_V_UPPER) - ) + return self.ast.add_node(ASTNode.char_class(cs_idx, negated)) # Word boundary anchors if ch == CHAR_B_LOWER: @@ -886,37 +915,16 @@ struct Parser[origin: Origin](Movable): # Unicode escapes: \uHHHH and \UHHHHHHHH if ch == CHAR_U_LOWER: var cp = self._parse_hex_digits(4) - if cp > 255 and not self.inline_flags.unicode(): - raise _regex_error( - ( - "Unicode code point > U+00FF needs UTF-8 mode" - " — prefix the pattern with (?u) or (*UTF8)" - ), - self.pos - 5, - ) + self._check_needs_unicode(cp, self.pos - 5) return self.ast.add_node(ASTNode.literal(cp)) if ch == CHAR_U_UPPER: var cp = self._parse_hex_digits(8) - if cp > 255 and not self.inline_flags.unicode(): - raise _regex_error( - ( - "Unicode code point > U+00FF needs UTF-8 mode" - " — prefix the pattern with (?u) or (*UTF8)" - ), - self.pos - 9, - ) + self._check_needs_unicode(cp, self.pos - 9) return self.ast.add_node(ASTNode.literal(cp)) - # Control character: \cX — PCRE/Perl formula: uppercase X, then - # XOR 0x40 (identical to & 0x1F for letters, differs for - # punctuation: \c{ is ';', \c; is '{'). + # Control character: \cX if ch == CHAR_C_LOWER: - if self._at_end(): - raise _regex_error("Expected character after \\c", self.pos - 1) - var ctrl = Int(self._advance()) - if ctrl >= 97 and ctrl <= 122: - ctrl -= 32 - return self.ast.add_node(ASTNode.literal(UInt32(ctrl ^ 0x40))) + return self.ast.add_node(ASTNode.literal(self._parse_control())) # Unicode properties: \p{L}, \P{Nd}, \p{Greek}. The ranges are # CODEPOINT ranges; UTF-8 mode compiles them to byte sequences, @@ -955,32 +963,6 @@ struct Parser[origin: Origin](Movable): var pidx = self.ast.add_charset(pcs^) return self.ast.add_node(ASTNode.char_class(pidx, False)) - # Shorthand character classes - if ch == CHAR_D_LOWER or ch == CHAR_D_UPPER: - var cs = CharSet.digit() - if ch == CHAR_D_UPPER: - cs.negate() - cs.build_bitmap() - var cs_idx = self.ast.add_charset(cs^) - var node = ASTNode.char_class(cs_idx, ch == CHAR_D_UPPER) - return self.ast.add_node(node^) - elif ch == CHAR_W_LOWER or ch == CHAR_W_UPPER: - var cs = CharSet.word() - if ch == CHAR_W_UPPER: - cs.negate() - cs.build_bitmap() - var cs_idx = self.ast.add_charset(cs^) - return self.ast.add_node( - ASTNode.char_class(cs_idx, ch == CHAR_W_UPPER) - ) - elif ch == CHAR_S_LOWER or ch == CHAR_S: - var cs = CharSet.whitespace() - if ch == CHAR_S: - cs.negate() - cs.build_bitmap() - var cs_idx = self.ast.add_charset(cs^) - return self.ast.add_node(ASTNode.char_class(cs_idx, ch == CHAR_S)) - # Literal character escapes if ch == CHAR_t: return self.ast.add_node(ASTNode.literal(UInt32(CHAR_TAB))) @@ -1032,24 +1014,7 @@ struct Parser[origin: Origin](Movable): # reading (each byte a member), which is what ROADMAP §3 # records as the byte-mode charset question. if self.inline_flags.unicode() and ch >= 0xC0: - var extra = 1 - if ch >= 0xF0: - extra = 3 - elif ch >= 0xE0: - extra = 2 - var cp: UInt32 - if ch >= 0xF0: - cp = UInt32(ch) & 0x07 - elif ch >= 0xE0: - cp = UInt32(ch) & 0x0F - else: - cp = UInt32(ch) & 0x1F - for _ in range(extra): - if self._at_end(): - break - var cont = self._advance() - cp = (cp << 6) | (UInt32(cont) & 0x3F) - return cp + return self._utf8_tail(ch) return UInt32(ch) # It is an escape @@ -1103,34 +1068,14 @@ struct Parser[origin: Origin](Movable): return self._parse_hex_digits(2) elif esc == CHAR_U_LOWER: var cp = self._parse_hex_digits(4) - if cp > 255 and not self.inline_flags.unicode(): - raise _regex_error( - ( - "Unicode code point > U+00FF needs UTF-8 mode" - " — prefix the pattern with (?u) or (*UTF8)" - ), - self.pos - 5, - ) + self._check_needs_unicode(cp, self.pos - 5) return cp elif esc == CHAR_U_UPPER: var cp = self._parse_hex_digits(8) - if cp > 255 and not self.inline_flags.unicode(): - raise _regex_error( - ( - "Unicode code point > U+00FF needs UTF-8 mode" - " — prefix the pattern with (?u) or (*UTF8)" - ), - self.pos - 9, - ) + self._check_needs_unicode(cp, self.pos - 9) return cp elif esc == CHAR_C_LOWER: - # Same PCRE/Perl formula as the atom path: uppercase, XOR 0x40. - if self._at_end(): - raise _regex_error("Expected character after \\c", self.pos - 1) - var ctrl = Int(self._advance()) - if ctrl >= 97 and ctrl <= 122: - ctrl -= 32 - return UInt32(ctrl ^ 0x40) + return self._parse_control() # Python's rule: an escaped ASCII letter or digit that is not a # recognized escape is an ERROR ([\p{L}] silently becoming the # literal set {p,{,L,}} was worse than a rejection). Escaped From 85116ddf5d039c5fdc855e89dff3204b5aaa12f2 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 11:59:10 -0600 Subject: [PATCH 02/65] cleanup: pcre2 compare reuses bench_compare/gen_pdf table code bench_compare_pcre2.py carried copies of print_comparison/speedup_bar and gen_pdf's build_table_data/generate_pdf differing only in labels, widths and titles. Parametrize the originals (defaults unchanged) and import them. Co-Authored-By: Claude Opus 5.5 (1M context) --- bench/bench_compare.py | 33 +++-- bench/bench_compare_pcre2.py | 239 +++-------------------------------- bench/gen_pdf.py | 40 +++--- 3 files changed, 57 insertions(+), 255 deletions(-) diff --git a/bench/bench_compare.py b/bench/bench_compare.py index f3a0cdd..f2a3971 100644 --- a/bench/bench_compare.py +++ b/bench/bench_compare.py @@ -88,10 +88,10 @@ def run_mojo_static_benchmarks() -> dict[str, float]: return _run_mojo_task("bench") -def speedup_bar(ratio: float) -> str: +def speedup_bar(ratio: float, cols: int = BAR_COLS) -> str: """Return a coloured bar string representing the speedup ratio.""" - filled = min(int(ratio / 10.0 * BAR_COLS), BAR_COLS) if ratio <= 10 else BAR_COLS - bar = "█" * filled + "░" * (BAR_COLS - filled) + filled = min(int(ratio / 10.0 * cols), cols) if ratio <= 10 else cols + bar = "█" * filled + "░" * (cols - filled) if ratio >= 1.0: return f"\033[32m{bar}\033[0m" # green = faster else: @@ -108,20 +108,25 @@ def _ratio_str(ratio: float) -> str: def print_comparison( py: dict[str, float], static: dict[str, float], + labels=("Python", "Static", "Py/Stat Bar (10x=full)"), + widths=(9, 9, 14, 50), # baseline col, Regex col, missing-ratio dash, rule + summary="Python vs Static — faster", + bar_cols=BAR_COLS, ): - """Print the two-column comparison table.""" + """Print the two-column comparison table; ratio = baseline ÷ Regex.""" + base_w, ours_w, dash_w, rule_w = widths all_names = list(py.keys()) if not all_names: - print(" No Python results collected.") + print(f" No {labels[0]} results collected.") return col_name = max(max(len(n) for n in all_names), 34) header = ( - f" {'Benchmark':<{col_name}} {'Python':>9} " - f"{'Static':>9} {'Py/Stat':>7} Bar (10x=full)" + f" {'Benchmark':<{col_name}} {labels[0]:>{base_w}} " + f"{labels[1]:>{ours_w}} {labels[2]}" ) - sep = " " + "─" * (col_name + 50) + sep = " " + "─" * (col_name + rule_w) print() print(header) @@ -133,30 +138,30 @@ def print_comparison( py_us = py[name] stat_us = static.get(name) - stat_str = f"{stat_us:>9.3f}" if stat_us is not None else f"{'—':>9}" + stat_str = f"{stat_us:>{ours_w}.3f}" if stat_us is not None else f"{'—':>{ours_w}}" if stat_us is not None and stat_us > 0: ratio = py_us / stat_us ratio_str = _ratio_str(ratio) - bar = speedup_bar(ratio) + bar = speedup_bar(ratio, bar_cols) if ratio >= 1.0: faster += 1 else: slower += 1 else: - ratio_str = f"{'—':>14}" - bar = speedup_bar(0) + ratio_str = f"{'—':>{dash_w}}" + bar = speedup_bar(0, bar_cols) if stat_us is None: missing += 1 print( - f" {name:<{col_name}} {py_us:>9.3f} " + f" {name:<{col_name}} {py_us:>{base_w}.3f} " f"{stat_str} {ratio_str} {bar}" ) print(sep) print( - f" Python vs Static — faster: {faster} | slower: {slower}" + f" {summary}: {faster} | slower: {slower}" + (f" | no data: {missing}" if missing else "") ) diff --git a/bench/bench_compare_pcre2.py b/bench/bench_compare_pcre2.py index 1b3b933..2cc2320 100644 --- a/bench/bench_compare_pcre2.py +++ b/bench/bench_compare_pcre2.py @@ -14,25 +14,13 @@ import argparse sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from bench_compare import run_mojo_static_benchmarks, _ratio_str as ratio_str +from bench_compare import run_mojo_static_benchmarks, print_comparison try: - from reportlab.lib import colors - from reportlab.lib.pagesizes import A4 - from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle - from reportlab.lib.units import cm - from reportlab.platypus import SimpleDocTemplate, Table, TableStyle, Paragraph, Spacer import gen_pdf except ImportError: pass - -# --------------------------------------------------------------------------- -# Config -# --------------------------------------------------------------------------- - -BAR_COLS = 16 # width of the speedup bar - # --------------------------------------------------------------------------- # Paths (repo-relative, resolved from this file's location) # --------------------------------------------------------------------------- @@ -117,214 +105,6 @@ def run_pcre2_benchmarks() -> dict[str, float]: return timings -# --------------------------------------------------------------------------- -# Display helpers -# --------------------------------------------------------------------------- - -def speedup_bar(ratio: float) -> str: - """Coloured █/░ bar: full at ratio=10.""" - filled = min(int(ratio / 10.0 * BAR_COLS), BAR_COLS) if ratio <= 10 else BAR_COLS - bar = "█" * filled + "░" * (BAR_COLS - filled) - color = "\033[32m" if ratio >= 1.0 else "\033[31m" - return f"{color}{bar}\033[0m" - - - - -def print_comparison( - pcre2: dict[str, float], - static: dict[str, float], -) -> None: - """Print a two-column comparison: PCRE2 JIT | Regex | ratio | bar. - - Ratio = PCRE2 time / Static time. - >1x means Regex is faster than PCRE2 JIT. - """ - # Use PCRE2 results as the canonical name list (it runs first/fully) - all_names = list(pcre2.keys()) - if not all_names: - print(" No PCRE2 results collected.") - return - - col_name = max(max(len(n) for n in all_names), 34) - - header = ( - f" {'Benchmark':<{col_name}} {'PCRE2 JIT':>10} " - f"{'Regex':>11} {'PCRE2/Stat':>10} Bar (PCRE2÷Static, 10x=full)" - ) - sep = " " + "─" * (col_name + 65) - - print() - print(header) - print(sep) - - faster = slower = missing = 0 - - for name in all_names: - pcre2_us = pcre2[name] - stat_us = static.get(name) - - pcre2_str = f"{pcre2_us:>10.3f}" - stat_str = f"{stat_us:>11.3f}" if stat_us is not None else f"{'—':>11}" - - if stat_us is not None and stat_us > 0: - ratio = pcre2_us / stat_us - r_str = ratio_str(ratio) - bar = speedup_bar(ratio) - if ratio >= 1.0: - faster += 1 - else: - slower += 1 - else: - r_str = f"{'—':>13}" - bar = speedup_bar(0) - missing += 1 - - print(f" {name:<{col_name}} {pcre2_str} {stat_str} {r_str} {bar}") - - print(sep) - print( - f" Regex faster: {faster} | slower: {slower}" - + (f" | no data: {missing}" if missing else "") - ) - - -# --------------------------------------------------------------------------- -# PDF Generation -# --------------------------------------------------------------------------- - -def build_pdf_table_data(pcre2: dict[str, float], static: dict[str, float]) -> tuple[list, list]: - """Build reportlab table and styles for PCRE2 vs Regex.""" - rows = [["Benchmark", "Regex (µs)", "PCRE2 JIT (µs)", "Ratio"]] - styles = [ - ("BACKGROUND", (0, 0), (-1, 0), gen_pdf.HEADER_BG), - ("TEXTCOLOR", (0, 0), (-1, 0), colors.white), - ("FONTNAME", (0, 0), (-1, 0), "Helvetica-Bold"), - ("FONTSIZE", (0, 0), (-1, 0), 9), - ("ALIGN", (0, 0), (-1, 0), "CENTER"), - ("BOTTOMPADDING", (0, 0), (-1, 0), 6), - ("TOPPADDING", (0, 0), (-1, 0), 6), - ("FONTNAME", (0, 1), (-1, -1), "Helvetica"), - ("FONTSIZE", (0, 1), (-1, -1), 8), - ("ALIGN", (1, 1), (-1, -1), "RIGHT"), - ("ALIGN", (0, 1), (0, -1), "LEFT"), - ("TOPPADDING", (0, 1), (-1, -1), 3), - ("BOTTOMPADDING", (0, 1), (-1, -1), 3), - ("ROWBACKGROUNDS", (0, 1), (-1, -1), [colors.white, colors.HexColor("#fafafa")]), - ("GRID", (0, 0), (-1, -1), 0.25, colors.HexColor("#cfd8dc")), - ] - - names = list(pcre2.keys()) - assignment = gen_pdf.assign_sections(names) - last_section = None - row_idx = 1 - faster = slower = 0 - - for name in names: - section = assignment[name] - if section != last_section: - rows.append([section, "", "", ""]) - styles += [ - ("BACKGROUND", (0, row_idx), (-1, row_idx), gen_pdf.SECTION_BG), - ("FONTNAME", (0, row_idx), (-1, row_idx), "Helvetica-Bold"), - ("FONTSIZE", (0, row_idx), (-1, row_idx), 8), - ("SPAN", (0, row_idx), (-1, row_idx)), - ("TOPPADDING", (0, row_idx), (-1, row_idx), 4), - ("BOTTOMPADDING", (0, row_idx), (-1, row_idx), 4), - ] - row_idx += 1 - last_section = section - - pcre_us = pcre2[name] - stat_us = static.get(name) - - if stat_us is None: - rows.append([name, "—", f"{pcre_us:.3f}", "—"]) - else: - ratio = pcre_us / stat_us - ratio_str = f"{ratio:.1f}x" - rows.append([name, f"{stat_us:.3f}", f"{pcre_us:.3f}", ratio_str]) - - if ratio >= 1.0: - faster += 1 - styles += [ - ("TEXTCOLOR", (3, row_idx), (3, row_idx), gen_pdf.GREEN), - ("FONTNAME", (3, row_idx), (3, row_idx), "Helvetica-Bold"), - ] - else: - slower += 1 - styles += [ - ("BACKGROUND", (0, row_idx), (-1, row_idx), gen_pdf.LIGHT_RED), - ("TEXTCOLOR", (3, row_idx), (3, row_idx), gen_pdf.RED), - ("FONTNAME", (3, row_idx), (3, row_idx), "Helvetica-Bold"), - ] - - row_idx += 1 - - rows.append([f"Regex faster: {faster} | slower: {slower}", "", "", ""]) - styles += [ - ("BACKGROUND", (0, row_idx), (-1, row_idx), gen_pdf.HEADER_BG), - ("TEXTCOLOR", (0, row_idx), (-1, row_idx), colors.white), - ("FONTNAME", (0, row_idx), (-1, row_idx), "Helvetica-Bold"), - ("FONTSIZE", (0, row_idx), (-1, row_idx), 8), - ("SPAN", (0, row_idx), (-1, row_idx)), - ("ALIGN", (0, row_idx), (-1, row_idx), "CENTER"), - ("TOPPADDING", (0, row_idx), (-1, row_idx), 5), - ("BOTTOMPADDING", (0, row_idx), (-1, row_idx), 5), - ] - return rows, styles - - -def generate_pdf(pcre2: dict[str, float], static: dict[str, float], output_path: str): - print(f"\n [pdf] Generating PDF → {output_path}...") - specs = gen_pdf.get_machine_specs() - - doc = SimpleDocTemplate( - output_path, pagesize=A4, leftMargin=1.5*cm, rightMargin=1.5*cm, - topMargin=1.5*cm, bottomMargin=1.5*cm - ) - - style_sheet = getSampleStyleSheet() - title_style = ParagraphStyle( - "Title", parent=style_sheet["Normal"], fontName="Helvetica-Bold", fontSize=16, - textColor=gen_pdf.HEADER_BG, spaceAfter=6 - ) - subtitle_style = ParagraphStyle( - "Subtitle", parent=style_sheet["Normal"], fontName="Helvetica", fontSize=10, - textColor=colors.HexColor("#546e7a"), spaceAfter=12 - ) - spec_style = ParagraphStyle( - "Spec", parent=style_sheet["Normal"], fontName="Helvetica", fontSize=8.5, - textColor=colors.HexColor("#37474f"), spaceAfter=2 - ) - - elements = [] - elements.append(Paragraph("Regex vs PCRE2 JIT — Benchmark Results", title_style)) - elements.append(Paragraph("Ratio = PCRE2 JIT ÷ Regex. >1x = Regex faster. JIT compile time excluded.", subtitle_style)) - - spec_rows = [[Paragraph(f"{k}", spec_style), Paragraph(v, spec_style)] for k, v in specs] - spec_table = Table(spec_rows, colWidths=[2.5*cm, 14*cm]) - spec_table.setStyle(TableStyle([ - ("FONTNAME", (0, 0), (-1, -1), "Helvetica"), - ("FONTSIZE", (0, 0), (-1, -1), 8.5), - ("TOPPADDING", (0, 0), (-1, -1), 2), - ("BOTTOMPADDING", (0, 0), (-1, -1), 2), - ("BACKGROUND", (0, 0), (-1, -1), colors.HexColor("#f5f5f5")), - ("GRID", (0, 0), (-1, -1), 0.25, colors.HexColor("#cfd8dc")), - ])) - elements.append(spec_table) - elements.append(Spacer(1, 0.4*cm)) - - table_rows, table_styles = build_pdf_table_data(pcre2, static) - table = Table(table_rows, colWidths=[7.5*cm, 2.8*cm, 2.8*cm, 2.0*cm], repeatRows=1) - table.setStyle(TableStyle(table_styles)) - elements.append(table) - - doc.build(elements) - print(" [pdf] PDF generation complete.") - - - # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- @@ -365,7 +145,14 @@ def main() -> None: print(f"\n{'═' * width}") print(f" Results (µs per operation)") print(f"{'═' * width}") - print_comparison(pcre2, static) + # Ratio = PCRE2 time / Regex time; >1x means Regex is faster than PCRE2 JIT. + print_comparison( + pcre2, static, + labels=("PCRE2 JIT", "Regex", "PCRE2/Stat Bar (PCRE2÷Static, 10x=full)"), + widths=(10, 11, 13, 65), + summary="Regex faster", + bar_cols=16, + ) if not pcre2: print("\n [note] PCRE2 data unavailable.") @@ -377,7 +164,13 @@ def main() -> None: print("\n [error] Cannot generate PDF: reportlab not installed.\n Try `pixi add reportlab` or similar.") else: output_pdf = os.path.join(REPO_ROOT, "bench_pcre2_results.pdf") - generate_pdf(pcre2, static, output_pdf) + print(f"\n [pdf] Generating PDF → {output_pdf}...") + gen_pdf.generate_pdf( + output_pdf, pcre2, static, "PCRE2 JIT", + "Regex vs PCRE2 JIT — Benchmark Results", + "Ratio = PCRE2 JIT ÷ Regex. >1x = Regex faster. JIT compile time excluded.", + ) + print(" [pdf] PDF generation complete.") print(f"\n{'═' * width}\n") diff --git a/bench/gen_pdf.py b/bench/gen_pdf.py index 3505119..903eb92 100644 --- a/bench/gen_pdf.py +++ b/bench/gen_pdf.py @@ -132,9 +132,10 @@ def assign_sections(names: list[str]) -> dict[str, str]: def build_table_data( py: dict[str, float], static_mojo: dict[str, float], + baseline_label: str = "Python", ) -> tuple[list, list]: - """Return (table_rows, style_commands).""" - rows = [["Benchmark", "Regex (µs)", "Python (µs)", "Ratio"]] + """Return (table_rows, style_commands); ratio = baseline ÷ Regex.""" + rows = [["Benchmark", "Regex (µs)", f"{baseline_label} (µs)", "Ratio"]] styles = [ # Header row ("BACKGROUND", (0, 0), (-1, 0), HEADER_BG), @@ -213,15 +214,15 @@ def build_table_data( return rows, styles -def generate_pdf(output_path: str): - print("Running benchmarks...") - py = run_python_benchmarks() - static_mojo = run_mojo_static_benchmarks() - - print("Getting machine specs...") +def generate_pdf( + output_path: str, + py: dict[str, float], + static_mojo: dict[str, float], + baseline_label: str = "Python", + title: str = "Regex vs Python re — Benchmark Results", + subtitle: str = "Ratio = Python time ÷ Regex time. >1x = Regex faster.", +): specs = get_machine_specs() - - print(f"Generating PDF → {output_path}") doc = SimpleDocTemplate( output_path, pagesize=A4, @@ -260,11 +261,8 @@ def generate_pdf(output_path: str): elements = [] # Title - elements.append(Paragraph("Regex vs Python re — Benchmark Results", title_style)) - elements.append(Paragraph( - "Ratio = Python time ÷ Regex time. >1x = Regex faster.", - subtitle_style, - )) + elements.append(Paragraph(title, title_style)) + elements.append(Paragraph(subtitle, subtitle_style)) # Machine specs table spec_rows = [[Paragraph(f"{k}", spec_style), Paragraph(v, spec_style)] for k, v in specs] @@ -282,14 +280,20 @@ def generate_pdf(output_path: str): # Benchmark table col_widths = [7.5 * cm, 2.8 * cm, 2.8 * cm, 2.0 * cm] - table_rows, table_styles = build_table_data(py, static_mojo) + table_rows, table_styles = build_table_data(py, static_mojo, baseline_label) table = Table(table_rows, colWidths=col_widths, repeatRows=1) table.setStyle(TableStyle(table_styles)) elements.append(table) doc.build(elements) - print(f"Done: {output_path}") if __name__ == "__main__": - generate_pdf(os.path.abspath(OUTPUT)) + out = os.path.abspath(OUTPUT) + print("Running benchmarks...") + py = run_python_benchmarks() + static_mojo = run_mojo_static_benchmarks() + print("Getting machine specs...") + print(f"Generating PDF → {out}") + generate_pdf(out, py, static_mojo) + print(f"Done: {out}") From 7570234591d4160fc3e016895d1b152f502812ab Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 11:59:34 -0600 Subject: [PATCH 03/65] cleanup: drop _negate_cp, reuse utf8.negate_ranges Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/nfa.mojo | 52 +++++--------------------------------- test/test_nfa_runtime.mojo | 2 +- 2 files changed, 8 insertions(+), 46 deletions(-) diff --git a/emberregex/nfa.mojo b/emberregex/nfa.mojo index c898628..54814a5 100644 --- a/emberregex/nfa.mojo +++ b/emberregex/nfa.mojo @@ -10,7 +10,12 @@ from std.math import max, min from .constants import CHAR_A_LOWER, CHAR_A_UPPER, CHAR_Z_LOWER, CHAR_Z_UPPER from .ast import AST, ASTNode, ASTNodeKind, AnchorKind from .charset import BITMAP_WIDTH, CharSet, CharRange -from .utf8 import UTF8_SEQ_LEN_SHIFT, UTF8_SEQ_WORDS, utf8_seq_table +from .utf8 import ( + UTF8_SEQ_LEN_SHIFT, + UTF8_SEQ_WORDS, + negate_ranges, + utf8_seq_table, +) from .flags import RegexFlags from .parser import parse from std.os import abort @@ -908,50 +913,7 @@ def _charset_codepoint_ranges(cs: CharSet) -> List[Int]: out.append(Int(r.lo)) out.append(Int(r.hi)) if cs.negated: - return _negate_cp(out) - return out^ - - -def _negate_cp(ranges: List[Int]) -> List[Int]: - var n = len(ranges) // 2 - var los = List[Int]() - var his = List[Int]() - for i in range(n): - los.append(ranges[2 * i]) - his.append(ranges[2 * i + 1]) - # Insertion sort, with the already-in-order case costing ONE comptime - # element access instead of five. The property tables are sorted and - # disjoint by construction (test_unicode_tables pins that for all 84), - # so every `\p{...}` takes the skip on every element; the original - # wrote `kl`/`kh` straight back over themselves 683 times for a - # `\p{L}`, and a comptime access is ~61 us. `prev` stays valid across - # an insertion: los[0..i] ends sorted, and `cur < prev` means its new - # maximum is still `prev`. - var prev = los[0] if n > 0 else 0 - for i in range(1, n): - var cur = los[i] - if cur >= prev: - prev = cur - continue - var kh = his[i] - var j = i - 1 - while j >= 0 and los[j] > cur: - los[j + 1] = los[j] - his[j + 1] = his[j] - j -= 1 - los[j + 1] = cur - his[j + 1] = kh - var out = List[Int]() - var cursor = 0 - for i in range(n): - if los[i] > cursor: - out.append(cursor) - out.append(los[i] - 1) - if his[i] + 1 > cursor: - cursor = Int(his[i]) + 1 - if cursor <= 0x10FFFF: - out.append(cursor) - out.append(0x10FFFF) + return negate_ranges(out) return out^ diff --git a/test/test_nfa_runtime.mojo b/test/test_nfa_runtime.mojo index 272578e..6824c7d 100644 --- a/test/test_nfa_runtime.mojo +++ b/test/test_nfa_runtime.mojo @@ -418,7 +418,7 @@ def test_unicode_negated_class_complements_codepoints() raises: assert_false(one.charsets[1].negated) # Out-of-order members (`x` before `a`) go through the insertion - # sort in `_negate_cp`; the complement is still ascending. + # sort in `negate_ranges`; the complement is still ascending. var two = _nfa("(?u)[^xa]") assert_equal(len(two.states), 41) assert_equal(len(two.charsets), 17) From d09e0bdc1a7362b5959b18fe015e75900a9ff66b Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 11:59:39 -0600 Subject: [PATCH 04/65] cleanup: @fieldwise_init for the Sbt* comptime structs Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/backtrack.mojo | 38 ++++---------------------------------- 1 file changed, 4 insertions(+), 34 deletions(-) diff --git a/emberregex/backtrack.mojo b/emberregex/backtrack.mojo index 40ab2fe..25f29a8 100644 --- a/emberregex/backtrack.mojo +++ b/emberregex/backtrack.mojo @@ -192,6 +192,7 @@ comptime SBT_GIVEBACK_POSSESSIVE = 2 body consumed can ever start it: the loop never gives anything back.""" +@fieldwise_init struct SbtLoopFilter(Copyable, Movable): """Comptime analysis of one simple loop: how far its giveback can be skipped, and the byte sets the walkers test against.""" @@ -203,16 +204,6 @@ struct SbtLoopFilter(Copyable, Movable): """`exit_bits | ~body`: bytes at which a lazy loop must stop scanning — either the exit could start there or the body can no longer consume.""" - def __init__( - out self, - mode: Int, - exit_bits: SIMD[DType.uint8, BITMAP_WIDTH], - stop_bits: SIMD[DType.uint8, BITMAP_WIDTH], - ): - self.mode = mode - self.exit_bits = exit_bits - self.stop_bits = stop_bits - def _sbt_loop_filter(nfa: NFA, body_idx: Int, exit_idx: Int) -> SbtLoopFilter: """Compile-time: derive a simple loop's giveback mode and byte sets. @@ -269,6 +260,7 @@ def sbt_loop_modes(nfa: NFA) -> List[Int]: return modes^ +@fieldwise_init struct SbtCounted(Copyable, Movable): """Comptime description of a counted repetition (`x{n,m}`) whose body is a single consuming state, rooted at one NFA state. @@ -288,22 +280,6 @@ struct SbtCounted(Copyable, Movable): var exit: Int var greedy: Bool - def __init__( - out self, - ok: Bool, - lo: Int, - hi: Int, - body: Int, - exit: Int, - greedy: Bool, - ): - self.ok = ok - self.lo = lo - self.hi = hi - self.body = body - self.exit = exit - self.greedy = greedy - def _sbt_is_body_state(nfa: NFA, idx: Int) -> Bool: """Compile-time: True when `idx` is a single consuming state — the only @@ -724,6 +700,7 @@ comptime SBT_STACK_BUDGET = 4 * 1024 * 1024 comptime SBT_STACK_RESERVE = 512 * 1024 +@fieldwise_init struct SbtStackBounds(Copyable, Movable): """This thread's stack, low and high address. `low == 0` means the platform could not be asked.""" @@ -731,10 +708,6 @@ struct SbtStackBounds(Copyable, Movable): var low: Int var high: Int - def __init__(out self, low: Int, high: Int): - self.low = low - self.high = high - @no_inline def sbt_stack_bounds() -> SbtStackBounds: @@ -879,6 +852,7 @@ def sbt_stack_floor[ return 0 +@fieldwise_init struct SbtDepthPlan(Copyable, Movable): """Where the walker has to watch the stack, decided at compile time.""" @@ -892,10 +866,6 @@ struct SbtDepthPlan(Copyable, Movable): entry to EVERY state instead: correct, and measurably slower, which is why it is computed rather than assumed.""" - def __init__(out self, needs_guard: Bool, splits_are_fvs: Bool): - self.needs_guard = needs_guard - self.splits_are_fvs = splits_are_fvs - def _sbt_depth_plan_list(nfa: NFA, cyclic: List[Bool]) -> SbtDepthPlan: """`sbt_depth_plan` over Lists — the reference semantics, and the From acb09d64abdc47fe2e7681fbf61dd389c44fa832 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:00:23 -0600 Subject: [PATCH 05/65] cleanup: drop dead imports in backtrack.mojo and executor.mojo Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/backtrack.mojo | 6 ------ emberregex/executor.mojo | 8 +------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/emberregex/backtrack.mojo b/emberregex/backtrack.mojo index 25f29a8..8d27c43 100644 --- a/emberregex/backtrack.mojo +++ b/emberregex/backtrack.mojo @@ -43,13 +43,8 @@ from std.sys.info import CompilationTarget from std.sys.intrinsics import llvm_intrinsic from .constants import ( - CHAR_A_LOWER, CHAR_A_UPPER, CHAR_NEWLINE, - CHAR_NINE, - CHAR_UNDERSCORE, - CHAR_ZERO, - CHAR_Z_LOWER, CHAR_Z_UPPER, is_word_byte, ) @@ -57,7 +52,6 @@ from .nfa import ( _build_static_nfa, split_cycle_flags, NFA, - NFAState, NFAStateKind, ) from .charset import BITMAP_WIDTH diff --git a/emberregex/executor.mojo b/emberregex/executor.mojo index ed33bce..545fa69 100644 --- a/emberregex/executor.mojo +++ b/emberregex/executor.mojo @@ -11,18 +11,12 @@ with restore-on-return to eliminate slot copying. """ from .constants import ( - CHAR_A_LOWER, CHAR_A_UPPER, CHAR_NEWLINE, - CHAR_NINE, - CHAR_UNDERSCORE, - CHAR_ZERO, - CHAR_Z_LOWER, CHAR_Z_UPPER, is_word_byte, ) -from .nfa import NFA, NFAState, NFAStateKind -from .charset import CharSet +from .nfa import NFA, NFAStateKind from .ast import AnchorKind from .result import MatchResult from std.collections import Array From d78f79eb99692f7b11af6644d1bfb5b67058fb04 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:00:29 -0600 Subject: [PATCH 06/65] cleanup: merge _build_star/_build_plus into _build_loop Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/nfa.mojo | 66 +++++++++++++-------------------------------- 1 file changed, 19 insertions(+), 47 deletions(-) diff --git a/emberregex/nfa.mojo b/emberregex/nfa.mojo index 54814a5..b9a956c 100644 --- a/emberregex/nfa.mojo +++ b/emberregex/nfa.mojo @@ -1125,10 +1125,10 @@ def _build_fragment( if not greedy: nfa.has_lazy = True - if min_rep == 0 and max_rep == -1: - return _build_star(nfa, ast, child_idx, greedy, flags) - elif min_rep == 1 and max_rep == -1: - return _build_plus(nfa, ast, child_idx, greedy, flags) + if max_rep == -1 and min_rep <= 1: + return _build_loop( + nfa, ast, child_idx, greedy, flags, at_least_one=min_rep == 1 + ) elif min_rep == 0 and max_rep == 1: return _build_question(nfa, ast, child_idx, greedy, flags) else: @@ -1265,47 +1265,18 @@ def _compute_fixed_length( return -1 -def _build_star( +def _build_loop( mut nfa: NFA, ast: AST, child_idx: Int, greedy: Bool, flags: RegexFlags, + at_least_one: Bool, ) raises -> NFAFragment: - """Build NFA fragment for * (zero or more).""" - var body = _build_fragment(nfa, ast, child_idx, flags) - var split_idx = nfa.add_state(NFAState(NFAStateKind.SPLIT)) - - ref state = nfa.states[split_idx] - - if greedy: - state.out1 = body.start # Prefer looping - state.out2 = -1 # Skip (dangling) - else: - state.out1 = -1 # Prefer skipping - state.out2 = body.start # Loop - - state.greedy = greedy - - # Patch body outputs back to the split state (loop) - nfa.patch(body, split_idx) - - var frag = NFAFragment(split_idx) - if greedy: - frag.add_out(split_idx, 2) # The skip edge is dangling - else: - frag.add_out(split_idx, 1) # The skip edge is dangling - return frag^ - - -def _build_plus( - mut nfa: NFA, - ast: AST, - child_idx: Int, - greedy: Bool, - flags: RegexFlags, -) raises -> NFAFragment: - """Build NFA fragment for + (one or more).""" + """Build NFA fragment for * (zero or more) or, with `at_least_one`, + + (one or more). Same states either way: the body loops back through + one SPLIT whose other edge is the dangling exit; `+` enters at the + body, `*` at the split.""" var body = _build_fragment(nfa, ast, child_idx, flags) var split_idx = nfa.add_state(NFAState(NFAStateKind.SPLIT)) @@ -1320,15 +1291,14 @@ def _build_plus( state.greedy = greedy - # Patch body outputs to the split state + # Patch body outputs back to the split state (loop) nfa.patch(body, split_idx) - # Fragment starts at the body, exits from the split - var frag = NFAFragment(body.start) + var frag = NFAFragment(body.start if at_least_one else split_idx) if greedy: - frag.add_out(split_idx, 2) + frag.add_out(split_idx, 2) # The exit edge is dangling else: - frag.add_out(split_idx, 1) + frag.add_out(split_idx, 1) # The exit edge is dangling return frag^ @@ -1416,10 +1386,12 @@ def _build_repetition( if max_rep == -1: # {n,} — required copies + star loop. n >= 2 here: the QUANTIFIER - # arm of _build_fragment sends `{0,}` to _build_star and `{1,}` to - # _build_plus, so a required copy always exists to patch. + # arm of _build_fragment sends `{0,}` and `{1,}` to _build_loop, + # so a required copy always exists to patch. assert has_result, "{n,} reached _build_repetition with n == 0" - var star = _build_star(nfa, ast, child_idx, greedy, flags) + var star = _build_loop( + nfa, ast, child_idx, greedy, flags, at_least_one=False + ) var patch_frag = NFAFragment(res_start) patch_frag.outs = res_outs.copy() patch_frag.out_slots = res_out_slots.copy() From 2c3afa8fb0b99c3ae9893bb61fe6f416f9d28b65 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:01:36 -0600 Subject: [PATCH 07/65] cleanup: drop dead negated param of ASTNode.char_class Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/ast.mojo | 5 +++-- emberregex/parser.mojo | 9 ++++----- test/test_parser_runtime.mojo | 8 +------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/emberregex/ast.mojo b/emberregex/ast.mojo index d36e67e..623e0ba 100644 --- a/emberregex/ast.mojo +++ b/emberregex/ast.mojo @@ -81,10 +81,11 @@ struct ASTNode(Copyable, Movable): return ASTNode(ASTNodeKind.DOT) @staticmethod - def char_class(charset_idx: Int, negated: Bool, out node: ASTNode): + def char_class(charset_idx: Int, out node: ASTNode): + """Negation lives on the pooled CharSet (`cs.negate()`), which is + all the NFA builder reads; `negated` stays False here.""" node = ASTNode(ASTNodeKind.CHAR_CLASS) node.charset_index = charset_idx - node.negated = negated @staticmethod def alternation(var children: List[Int], out node: ASTNode): diff --git a/emberregex/parser.mojo b/emberregex/parser.mojo index 83c6a36..d830aef 100644 --- a/emberregex/parser.mojo +++ b/emberregex/parser.mojo @@ -800,12 +800,11 @@ struct Parser[origin: Origin](Movable): cs = CharSet.word() else: cs = CharSet.whitespace() - var negated = ch <= CHAR_Z_UPPER # the uppercase forms - if negated: + if ch <= CHAR_Z_UPPER: # the uppercase forms cs.negate() cs.build_bitmap() var cs_idx = self.ast.add_charset(cs^) - return self.ast.add_node(ASTNode.char_class(cs_idx, negated)) + return self.ast.add_node(ASTNode.char_class(cs_idx)) # Word boundary anchors if ch == CHAR_B_LOWER: @@ -961,7 +960,7 @@ struct Parser[origin: Origin](Movable): ) pcs.build_bitmap() var pidx = self.ast.add_charset(pcs^) - return self.ast.add_node(ASTNode.char_class(pidx, False)) + return self.ast.add_node(ASTNode.char_class(pidx)) # Literal character escapes if ch == CHAR_t: @@ -1341,7 +1340,7 @@ struct Parser[origin: Origin](Movable): cs.negate() var cs_idx = self.ast.add_charset(cs^) - return self.ast.add_node(ASTNode.char_class(cs_idx, negated)) + return self.ast.add_node(ASTNode.char_class(cs_idx)) def parse(pattern: String) raises -> AST: diff --git a/test/test_parser_runtime.mojo b/test/test_parser_runtime.mojo index 957d2f9..244a4fd 100644 --- a/test/test_parser_runtime.mojo +++ b/test/test_parser_runtime.mojo @@ -418,12 +418,10 @@ def test_parse_anchors() raises: def test_parse_escape_classes() raises: # \h \H \v \V (PCRE classes) and \d \D \w \W \s \S: the negated forms - # negate the CharSet itself (which is what the NFA builder reads) and - # mirror it on the node. + # negate the CharSet itself (which is what the NFA builder reads). def cls(pattern: String, negated: Bool) raises -> AST: var a = parse(pattern) assert_equal(_kind(a, a.root), ASTNodeKind.CHAR_CLASS) - assert_equal(a.nodes[a.root].negated, negated) assert_equal(a.charsets[a.nodes[a.root].charset_index].negated, negated) return a^ @@ -466,13 +464,11 @@ def test_parse_literal_escapes() raises: def test_parse_unicode_property() raises: var a = parse("\\p{Nd}") assert_equal(_kind(a, a.root), ASTNodeKind.CHAR_CLASS) - assert_false(a.nodes[a.root].negated) assert_false(a.charsets[0].negated) # ASCII digits, an Arabic-Indic digit (above the bitmap), no letters. _assert_class(a, a.root, [48, 57, 0x663], [47, 58, 97, 0x3B1]) # \P{...} negates the RANGES (a positive charset), not the flag. var n = parse("\\P{Nd}") - assert_false(n.nodes[n.root].negated) assert_false(n.charsets[0].negated) _assert_class(n, n.root, [47, 58, 97, 0x3B1], [48, 57, 0x663]) assert_equal( @@ -757,7 +753,6 @@ def test_parse_class_items() raises: assert_equal(_nranges(a, a.root), 1) _assert_class(a, a.root, [97, 98, 99], [96, 100]) a = parse("[^a]") - assert_true(a.nodes[a.root].negated) assert_true(a.charsets[0].negated) _assert_class(a, a.root, [98, 0x100], [97]) assert_equal( @@ -795,7 +790,6 @@ def test_parse_class_shorthands() raises: def cls(pattern: String) raises -> AST: var a = parse(pattern) assert_equal(_kind(a, a.root), ASTNodeKind.CHAR_CLASS) - assert_false(a.nodes[a.root].negated) assert_false(a.charsets[0].negated) return a^ From 7250e1a9ff72eb5e4f9f68c16c3be5e5260a671b Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:01:51 -0600 Subject: [PATCH 08/65] cleanup: one anchor check for Pike VM and heap backtracker, drop word-char forwarders PikeVM._check_anchor/_is_word_char duplicated _bt_check_anchor; the _bt_/_sbt_is_word_char forwarders become is_word_byte, and the WORD / NOT_WORD arms share one body. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/backtrack.mojo | 28 ++++----------- emberregex/executor.mojo | 75 +++++---------------------------------- 2 files changed, 15 insertions(+), 88 deletions(-) diff --git a/emberregex/backtrack.mojo b/emberregex/backtrack.mojo index 8d27c43..1fc10c2 100644 --- a/emberregex/backtrack.mojo +++ b/emberregex/backtrack.mojo @@ -66,11 +66,6 @@ from .simd_kernels import ( ) -@always_inline -def _sbt_is_word_char(ch: Byte) -> Bool: - return is_word_byte(ch) - - @always_inline def _sbt_to_lower(ch: Byte) -> Byte: if ch >= CHAR_A_UPPER and ch <= CHAR_Z_UPPER: @@ -109,22 +104,13 @@ def _sbt_check_anchor[ return pos == input_len elif anchor_type == AnchorKind.EOL_MULTILINE: return pos == input_len or input.unsafe_get(pos) == CHAR_NEWLINE - elif anchor_type == AnchorKind.WORD_BOUNDARY: - var left_is_word = False - var right_is_word = False - if pos > 0: - left_is_word = _sbt_is_word_char(input.unsafe_get(pos - 1)) - if pos < input_len: - right_is_word = _sbt_is_word_char(input.unsafe_get(pos)) - return left_is_word != right_is_word - elif anchor_type == AnchorKind.NOT_WORD_BOUNDARY: - var left_is_word = False - var right_is_word = False - if pos > 0: - left_is_word = _sbt_is_word_char(input.unsafe_get(pos - 1)) - if pos < input_len: - right_is_word = _sbt_is_word_char(input.unsafe_get(pos)) - return left_is_word == right_is_word + elif ( + anchor_type == AnchorKind.WORD_BOUNDARY + or anchor_type == AnchorKind.NOT_WORD_BOUNDARY + ): + var left = pos > 0 and is_word_byte(input.unsafe_get(pos - 1)) + var right = pos < input_len and is_word_byte(input.unsafe_get(pos)) + return (left != right) == (anchor_type == AnchorKind.WORD_BOUNDARY) return False diff --git a/emberregex/executor.mojo b/emberregex/executor.mojo index 545fa69..9dd9acf 100644 --- a/emberregex/executor.mojo +++ b/emberregex/executor.mojo @@ -432,7 +432,7 @@ struct PikeVM[num_slots: Int](Copyable): elif kind == NFAStateKind.ANCHOR: gen.unsafe_set(state_idx, gen_val) - if self._check_anchor(state.anchor_type, input, input_len, pos): + if _bt_check_anchor(state.anchor_type, input, input_len, pos): state_idx = state.out1 continue return @@ -508,52 +508,6 @@ struct PikeVM[num_slots: Int](Copyable): slot_data.append(slots.unsafe_get(s)) return - def _check_anchor[ - origin: Origin, // - ]( - self, - anchor_type: Int, - input: Span[Byte, origin], - input_len: Int, - pos: Int, - ) -> Bool: - """Check if an anchor assertion holds at the given position. - - MULTILINE behavior is baked into the anchor kind at NFA construction time: - BOL_MULTILINE / EOL_MULTILINE handle line-boundary matching without a runtime flag check. - """ - var ptr = Pointer(input.unsafe_ptr()) - if anchor_type == AnchorKind.BOL: - return pos == 0 - elif anchor_type == AnchorKind.BOL_MULTILINE: - return pos == 0 or input.unsafe_get(pos - 1) == CHAR_NEWLINE - elif anchor_type == AnchorKind.EOL: - return pos == input_len - elif anchor_type == AnchorKind.EOL_MULTILINE: - return pos == input_len or input.unsafe_get(pos) == CHAR_NEWLINE - elif anchor_type == AnchorKind.WORD_BOUNDARY: - var before_word = pos > 0 and Self._is_word_char( - ptr.unsafe_offset(pos - 1).unsafe_load() - ) - var after_word = pos < input_len and Self._is_word_char( - ptr.unsafe_offset(pos).unsafe_load() - ) - return before_word != after_word - elif anchor_type == AnchorKind.NOT_WORD_BOUNDARY: - var before_word = pos > 0 and Self._is_word_char( - ptr.unsafe_offset(pos - 1).unsafe_load() - ) - var after_word = pos < input_len and Self._is_word_char( - ptr.unsafe_offset(pos).unsafe_load() - ) - return before_word == after_word - return False - - @staticmethod - def _is_word_char(ch: Byte) -> Bool: - """Check if a character is a word character [a-zA-Z0-9_].""" - return is_word_byte(ch) - def _bt_try_match[ origin: Origin, // @@ -852,29 +806,16 @@ def _bt_check_anchor[ return pos == input_len elif anchor_type == AnchorKind.EOL_MULTILINE: return pos == input_len or input.unsafe_get(pos) == CHAR_NEWLINE - elif anchor_type == AnchorKind.WORD_BOUNDARY: - var left_is_word = False - var right_is_word = False - if pos > 0: - left_is_word = _bt_is_word_char(input.unsafe_get(pos - 1)) - if pos < input_len: - right_is_word = _bt_is_word_char(input.unsafe_get(pos)) - return left_is_word != right_is_word - elif anchor_type == AnchorKind.NOT_WORD_BOUNDARY: - var left_is_word = False - var right_is_word = False - if pos > 0: - left_is_word = _bt_is_word_char(input.unsafe_get(pos - 1)) - if pos < input_len: - right_is_word = _bt_is_word_char(input.unsafe_get(pos)) - return left_is_word == right_is_word + elif ( + anchor_type == AnchorKind.WORD_BOUNDARY + or anchor_type == AnchorKind.NOT_WORD_BOUNDARY + ): + var left = pos > 0 and is_word_byte(input.unsafe_get(pos - 1)) + var right = pos < input_len and is_word_byte(input.unsafe_get(pos)) + return (left != right) == (anchor_type == AnchorKind.WORD_BOUNDARY) return False -def _bt_is_word_char(ch: Byte) -> Bool: - return is_word_byte(ch) - - def _bt_to_lower(ch: Byte) -> Byte: if ch >= CHAR_A_UPPER and ch <= CHAR_Z_UPPER: return ch + 32 From 9ccbb99ed534c4d8bf8d7c3a0cc79fa63dad81d3 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:02:03 -0600 Subject: [PATCH 09/65] cleanup: drop edfa_is_word, WIDE_TABLE_CAP and _ShengState aliases Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/onepass.mojo | 9 ++++---- emberregex/sheng.mojo | 32 +++++++++++++---------------- emberregex/simd_kernels.mojo | 4 ---- emberregex/static_dfa.mojo | 11 ++-------- emberregex/static_rdfa.mojo | 7 +++---- test/test_dfa_builders_runtime.mojo | 7 +++---- 6 files changed, 26 insertions(+), 44 deletions(-) diff --git a/emberregex/onepass.mojo b/emberregex/onepass.mojo index 4f565c9..837b0d8 100644 --- a/emberregex/onepass.mojo +++ b/emberregex/onepass.mojo @@ -85,7 +85,7 @@ from std.sys import simd_width_of from .ast import AnchorKind from .backtrack import _sbt_is_simple_body -from .constants import CHAR_NEWLINE +from .constants import CHAR_NEWLINE, is_word_byte from .nfa import NFA, NFAStateKind, split_cycle_flags from .static_bytes import filled_string from .static_dfa import ( @@ -101,7 +101,6 @@ from .static_dfa import ( _is_word_byte, _nfa_has_word_anchor, _wb_holds, - edfa_is_word, ) from .simd_kernels import ( ACCEL_SHUFTI, @@ -783,7 +782,7 @@ def _op_start_state[op: OnePass](input: Span[Byte, _], start: Int) -> Int: if b == CHAR_NEWLINE: return op.start_nl comptime if op.start_word != op.start_other: - if edfa_is_word(b): + if is_word_byte(b): return op.start_word return op.start_other @@ -809,13 +808,13 @@ def _op_match_ok[ ): return False if flags & Int(OP_NEED_WORD) != 0 and ( - at_eof or not edfa_is_word(input.unsafe_get(end_pin)) + at_eof or not is_word_byte(input.unsafe_get(end_pin)) ): return False if ( flags & Int(OP_NEED_NONWORD) != 0 and not at_eof - and edfa_is_word(input.unsafe_get(end_pin)) + and is_word_byte(input.unsafe_get(end_pin)) ): return False return True diff --git a/emberregex/sheng.mojo b/emberregex/sheng.mojo index 75ccb1b..615a97e 100644 --- a/emberregex/sheng.mojo +++ b/emberregex/sheng.mojo @@ -39,7 +39,7 @@ search_forward here any more. from std.collections import Array from std.sys import simd_width_of -from .constants import CHAR_NEWLINE +from .constants import CHAR_NEWLINE, is_word_byte from .static_bytes import filled_string from .static_dfa import ( EDFA_EOL_AT_END, @@ -50,12 +50,11 @@ from .static_dfa import ( _edfa_has_accel, _edfa_has_region, _edfa_region_skip, - edfa_is_word, ) from .simd_kernels import ( + HAS_WIDE_BYTE_SHUFFLE, NIBBLE_TABLE_SIZE, - SHUFFLE_INDEX_LANES, - WIDE_TABLE_CAP, + _ShuffleIndex, nibble_lookup, table_lookup_32, table_lookup_64, @@ -63,12 +62,7 @@ from .simd_kernels import ( # Widest tbl tier this target can do in one instruction (see module # docstring) — an algorithmic constant, NOT a platform vector width. -comptime SHENG_STATE_CAP = WIDE_TABLE_CAP - -# The state vector broadcasts one state id across the shuffle's index -# register; only lane 0 is ever read back. Its width is the tbl result -# width and is independent of the mask width. -comptime _ShengState = SIMD[DType.uint8, SHUFFLE_INDEX_LANES] +comptime SHENG_STATE_CAP = 64 if HAS_WIDE_BYTE_SHUFFLE else NIBBLE_TABLE_SIZE # Flat 256 x cap mask table. Scalar element type matters: comptime # Array[Int32/UInt8, n] parameters lower to shared constant data in @@ -150,8 +144,10 @@ def sheng_masks_str[cap: Int](d: EagerDFA, enabled: Bool) -> String: @always_inline def _sheng_step[ cap: Int -](masks: StringLiteral, b: Byte, state_vec: _ShengState) -> _ShengState: - """One transition: shuffle the byte's mask by the state vector. +](masks: StringLiteral, b: Byte, state_vec: _ShuffleIndex) -> _ShuffleIndex: + """One transition: shuffle the byte's mask by the state vector (the + state id broadcast across the index register; only lane 0 is ever + read back, and its width is independent of the mask width). Each branch loads and shuffles at the literal tier width `cap`: only the tier this DFA needs is emitted, and the NEON-only tiers are never elaborated where cap is always @@ -279,7 +275,7 @@ def _sheng_full_match_impl[ # `masks` / `flags` are comptime arrays; `materialize` binds them to the # constant data emitted in the binary (no copy) so the walk can index them. var flg = materialize[flags]() - var cur_vec = _ShengState(UInt8(d.start_at_0)) + var cur_vec = _ShuffleIndex(UInt8(d.start_at_0)) var cur = d.start_at_0 var pos = 0 var input_len = len(input) @@ -291,7 +287,7 @@ def _sheng_full_match_impl[ var before = cur skipped = _edfa_region_skip[d=d](input, cur, skipped) if cur != before: - cur_vec = _ShengState(UInt8(cur)) + cur_vec = _ShuffleIndex(UInt8(cur)) pos = skipped if pos >= input_len: break @@ -383,12 +379,12 @@ def _sheng_walk_impl[ cur = s_nl else: comptime if s_other_w != s_other: - cur = s_other_w if edfa_is_word( + cur = s_other_w if is_word_byte( input.unsafe_get(start - 1) ) else s_other else: cur = s_other - var cur_vec = _ShengState(UInt8(cur)) + var cur_vec = _ShuffleIndex(UInt8(cur)) var last_match = -1 if cur < d.num_match_states: @@ -403,7 +399,7 @@ def _sheng_walk_impl[ var before = cur skipped = _edfa_region_skip[d=d](input, cur, skipped) if cur != before: - cur_vec = _ShengState(UInt8(cur)) + cur_vec = _ShuffleIndex(UInt8(cur)) pos = skipped if pos >= input_len: break @@ -417,7 +413,7 @@ def _sheng_walk_impl[ comptime if d.any_wb: if UInt(cur - d.num_match_states) < UInt(d.num_cond_states): var f = flg.unsafe_get(cur) - if ((f & EDFA_MATCH_IF_WORD) != 0) == edfa_is_word(b): + if ((f & EDFA_MATCH_IF_WORD) != 0) == is_word_byte(b): last_match = pos cur_vec = _sheng_step[cap](masks, b, cur_vec) cur = Int(cur_vec[0]) diff --git a/emberregex/simd_kernels.mojo b/emberregex/simd_kernels.mojo index ae3b325..3132045 100644 --- a/emberregex/simd_kernels.mojo +++ b/emberregex/simd_kernels.mojo @@ -51,10 +51,6 @@ comptime HAS_FAST_BYTE_SHUFFLE = ( # executes once per input byte. comptime HAS_WIDE_BYTE_SHUFFLE = CompilationTarget.has_neon() -# Widest single-instruction table lookup on this target. This is the -# ceiling on Sheng's state count, not a vector width. -comptime WIDE_TABLE_CAP = 64 if HAS_WIDE_BYTE_SHUFFLE else NIBBLE_TABLE_SIZE - # One tbl/pshufb produces 16 result bytes, so the index vector is one # 128-bit register no matter how wide the table is. comptime SHUFFLE_INDEX_LANES = 16 diff --git a/emberregex/static_dfa.mojo b/emberregex/static_dfa.mojo index e31e0de..ddf467d 100644 --- a/emberregex/static_dfa.mojo +++ b/emberregex/static_dfa.mojo @@ -159,13 +159,6 @@ def _is_word_byte(b: Int) -> Bool: return b >= 0 and b < 256 and is_word_byte(Byte(b)) -@always_inline -def edfa_is_word(b: Byte) -> Bool: - """Runtime twin of `_is_word_byte`: the shared `constants.is_word_byte` - every engine's `\\b` check uses.""" - return is_word_byte(b) - - def _wb_holds(anchor_kind: Int, prev_word: Bool, next_word: Bool) -> Bool: """Comptime: does a word-boundary anchor of `anchor_kind` hold between a byte of class `prev_word` and one of class `next_word`? Out of @@ -2416,7 +2409,7 @@ def _edfa_walk_impl[ cur = s_nl else: comptime if s_other_w != s_other: - cur = s_other_w if edfa_is_word( + cur = s_other_w if is_word_byte( input.unsafe_get(start - 1) ) else s_other else: @@ -2448,7 +2441,7 @@ def _edfa_walk_impl[ # Such states occupy one id range (see num_cond_states). if UInt(cur - d.num_match_states) < UInt(d.num_cond_states): var f = flg.unsafe_get(cur) - if ((f & EDFA_MATCH_IF_WORD) != 0) == edfa_is_word(b): + if ((f & EDFA_MATCH_IF_WORD) != 0) == is_word_byte(b): last_match = pos var nxt = Int(tbl[unsafe_offset=cur * 256 + Int(b)]) if nxt < 0: diff --git a/emberregex/static_rdfa.mojo b/emberregex/static_rdfa.mojo index 835a223..4876646 100644 --- a/emberregex/static_rdfa.mojo +++ b/emberregex/static_rdfa.mojo @@ -48,7 +48,7 @@ from std.collections import Array from std.sys import simd_width_of from .ast import AnchorKind -from .constants import CHAR_NEWLINE +from .constants import CHAR_NEWLINE, is_word_byte from .nfa import NFA, NFAStateKind from .set_reverse import _reverse_edges, _rev_flat_closure from .simd_kernels import ( @@ -84,7 +84,6 @@ from .static_dfa import ( _word_anchor_bits, WB_PENDING, WB_RESOLVE, - edfa_is_word, ) # Per-state accept bits. @@ -870,7 +869,7 @@ def rdfa_find_start[ cur = d.seed_at_nl else: comptime if d.seed_other_word != d.seed_other: - cur = d.seed_other_word if edfa_is_word( + cur = d.seed_other_word if is_word_byte( input.unsafe_get(end) ) else d.seed_other else: @@ -897,7 +896,7 @@ def rdfa_find_start[ best = pos comptime if d.any_wb: if (f & (RDFA_WB_LEFT_WORD | RDFA_WB_LEFT_NONWORD)) != 0: - if ((f & RDFA_WB_LEFT_WORD) != 0) == edfa_is_word(b): + if ((f & RDFA_WB_LEFT_WORD) != 0) == is_word_byte(b): best = pos if pos <= floor: return best diff --git a/test/test_dfa_builders_runtime.mojo b/test/test_dfa_builders_runtime.mojo index 8a36a3b..5af4add 100644 --- a/test/test_dfa_builders_runtime.mojo +++ b/test/test_dfa_builders_runtime.mojo @@ -19,7 +19,7 @@ answer). Nothing here selects a lane, so no lane pin applies. from emberregex.nfa import build_nfa, NFA from emberregex.parser import parse -from emberregex.constants import CHAR_NEWLINE +from emberregex.constants import CHAR_NEWLINE, is_word_byte from emberregex.simd_kernels import ( ACCEL_SHUFTI, ACCEL_TRUFFLE, @@ -30,7 +30,6 @@ from emberregex.static_dfa import ( _minimize, EDFA_DEAD, EDFA_NFA_CAP, - edfa_is_word, ) from emberregex.static_rdfa import ( build_reverse_dfa, @@ -78,7 +77,7 @@ def _find_start(d: RDFA, text: String, end: Int, floor: Int = 0) -> Int: cur = d.seed_at_end elif input[end] == CHAR_NEWLINE: cur = d.seed_at_nl - elif edfa_is_word(input[end]): + elif is_word_byte(input[end]): cur = d.seed_other_word else: cur = d.seed_other @@ -96,7 +95,7 @@ def _find_start(d: RDFA, text: String, end: Int, floor: Int = 0) -> Int: if (f & RDFA_BOLNL) != 0 and b == CHAR_NEWLINE: best = pos if (f & (RDFA_WB_LEFT_WORD | RDFA_WB_LEFT_NONWORD)) != 0: - if ((f & RDFA_WB_LEFT_WORD) != 0) == edfa_is_word(b): + if ((f & RDFA_WB_LEFT_WORD) != 0) == is_word_byte(b): best = pos if pos <= floor: return best From 7e2f43f57610edad12a18c33cd2f0fa9b2536276 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:02:10 -0600 Subject: [PATCH 10/65] cleanup: drop run_coverage.py --gcov pipeline A one-off cross-check CI never runs (--check-baseline refused it). Removes parse_gcov_intermediate, the gcov branches in cover_one, the flag and its guard, the opt/llvm-cov lookup and TestGcovParse. compiler-rt stays: the default link pulls libclang_rt.osx.a (clang -### shows it), and clang_impl depends on it anyway; its pixi.toml comment now says so. Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 2 +- pixi.toml | 8 +-- run_coverage.py | 124 +++++++++---------------------------- tools/test_run_coverage.py | 12 ---- 4 files changed, 35 insertions(+), 111 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index cf851e7..2e200ee 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co pixi run test # run all tests (incremental: skips unchanged green files; --all forces) pixi run bench # single-pattern (`Regex`) benchmark suite pixi run bench_all # run all benchmarks -pixi run coverage # line coverage of emberregex/ over the suite (runs in the `cov` env, conda-forge LLVM; inline-aware counters by default, --gcov for the plain gcov pass; --missing, --opt-level, see run_coverage.py) +pixi run coverage # line coverage of emberregex/ over the suite (runs in the `cov` env, conda-forge LLVM; inline-aware counters; --missing, --opt-level, see run_coverage.py) pixi run coverage --check-baseline # the CI gate: fail if coverage falls below coverage-baseline.json pixi run coverage --update-baseline # rewrite that baseline from this run (commit it, and say why in the PR) pixi build # build the conda package (pixi-build-mojo -> lib/mojo/emberregex.mojoc) diff --git a/pixi.toml b/pixi.toml index ed07319..82612c9 100644 --- a/pixi.toml +++ b/pixi.toml @@ -54,10 +54,10 @@ format = "mojo format emberregex test bench" mojo = "==1.1.0rc0" [feature.cov.dependencies] -# One LLVM for opt/llc/llvm-cov (llvm-tools), the link driver (clang), the -# gcov runtime (compiler-rt) and the linker (lld: Apple's ld asserts on -# Mojo's long symbol names). Mojo itself is LLVM 24 trunk; the closest -# release keeps the IR parseable. +# One LLVM for llc (llvm-tools), the link driver (clang), the builtins +# archive clang links into every binary (compiler-rt: libclang_rt.osx.a) +# and the linker (lld: Apple's ld asserts on Mojo's long symbol names). +# Mojo itself is LLVM 24 trunk; the closest release keeps the IR parseable. llvm-tools = "23.*" clang = "23.*" compiler-rt = "23.*" diff --git a/run_coverage.py b/run_coverage.py index 4d2b8cd..94932ea 100644 --- a/run_coverage.py +++ b/run_coverage.py @@ -27,15 +27,6 @@ binary runs (pass/fail judged exactly as run_test.py does) and the counter file is read back against the site table. -`--gcov` runs the reference pipeline instead — `opt -passes= -insert-gcov-profiling`, `clang --coverage`, `llvm-cov gcov -i` — as a -cross-check for non-inlined code (the two agree on every line both -instrument). Mojo emits two compile units (the test file, and `` -holding the library and stdlib); the gcov pass instruments the module -once per unit, so the FIRST unit's files carry every function and the -second's `.gcda` is rejected by llvm-cov ("Invalid .gcda File!" on -stderr), contributing nothing to the union. - Line hits are unioned across all test binaries (a line is covered when any binary executed it) and reported per library source. Only RUNTIME code is visible: everything that executes in the comptime interpreter @@ -49,7 +40,7 @@ ratio CI ratchets against: `--check-baseline` fails a run that falls below it, `--update-baseline` rewrites it. -Usage: python3 run_coverage.py [-j N] [--only SUBSTR] [--missing] [--gcov] +Usage: python3 run_coverage.py [-j N] [--only SUBSTR] [--missing] [--opt-level N] [--llvm-bin DIR] [--keep] [--no-pkg] [--no-skip] [--check-baseline | --update-baseline] python3 run_coverage.py instrument SRC.ll DST.ll DUMP SITES.json @@ -75,7 +66,7 @@ DUMP_C = os.path.join(ROOT, "tools", "cov_dump.c") LIB_PREFIX = "emberregex" + os.sep MOJO_RUNTIME_LIB = "KGENCompilerRTShared" -STAGES = ("emit", "instrument", "opt", "llc", "link", "run", "collect") +STAGES = ("emit", "instrument", "llc", "link", "run", "collect") # --- Inline-aware instrumentation (pure; tools/test_run_coverage.py) -------- @@ -263,27 +254,6 @@ def instrument_file(src, dst, dump, sites_path): json.dump(inst.sites, f) -# --- gcov intermediate format (pure) ---------------------------------------- - - -def parse_gcov_intermediate(text): - """`llvm-cov gcov -i` output: `file:` headers followed by - `lcount:,` records (`function:`/`branch:` ignored). - Returns {path: {line: count}}; a line repeated within one file (one - entry per function covering it) sums.""" - out = {} - cur = None - for ln in text.splitlines(): - if ln.startswith("file:"): - cur = ln[5:].strip() - out.setdefault(cur, {}) - elif ln.startswith("lcount:") and cur is not None: - parts = ln[7:].split(",") - n, c = int(parts[0]), int(parts[1]) - out[cur][n] = out[cur].get(n, 0) + c - return out - - # --- Aggregation and report (pure) ------------------------------------------ @@ -464,7 +434,7 @@ def total(self): return sum(self.stages.values()) -def cover_one(path, flags, tools, lib_dir, opt_level, gcov, keep): +def cover_one(path, flags, tools, lib_dir, opt_level, keep): name = os.path.splitext(os.path.basename(path))[0] d = os.path.join(COV_DIR, name) shutil.rmtree(d, ignore_errors=True) @@ -481,7 +451,6 @@ def step(stage, cmd, cwd=ROOT): ll = os.path.join(d, name + ".ll") ll2 = os.path.join(d, name + ".cov.ll") - bc = os.path.join(d, name + ".bc") obj = os.path.join(d, name + ".o") exe = os.path.join(d, name) dump = os.path.join(d, "counters.bin") @@ -490,26 +459,16 @@ def step(stage, cmd, cwd=ROOT): if step("emit", ["mojo", "build", "--emit", "llvm", "--debug-level", "line-tables", f"-O{opt_level}", *flags, path, "-o", ll]).returncode: return out - if gcov: - # cwd = d: the pass bakes `/.gcda` into the binary as - # the path the runtime writes to, and drops the .gcno beside it. - if step("opt", [tools["opt"], "-passes=insert-gcov-profiling", ll, "-o", bc], cwd=d).returncode: - return out - codegen_in = bc - else: - # A subprocess, not a call: the pool is threads, and six Python - # passes over 100 MB of IR would serialize on the GIL while the - # compilers around them run in parallel. - if step("instrument", [sys.executable, os.path.abspath(__file__), "instrument", - ll, ll2, dump, sites_path]).returncode: - return out - codegen_in = ll2 - if step("llc", [tools["llc"], "-O0", "-filetype=obj", codegen_in, "-o", obj]).returncode: + # A subprocess, not a call: the pool is threads, and six Python + # passes over 100 MB of IR would serialize on the GIL while the + # compilers around them run in parallel. + if step("instrument", [sys.executable, os.path.abspath(__file__), "instrument", + ll, ll2, dump, sites_path]).returncode: return out - link = [tools["clang"], "-fuse-ld=lld", obj, "-L", lib_dir, "-l" + MOJO_RUNTIME_LIB, - "-Wl,-rpath," + lib_dir, "-o", exe] - link += ["--coverage"] if gcov else [DUMP_C] - if step("link", link).returncode: + if step("llc", [tools["llc"], "-O0", "-filetype=obj", ll2, "-o", obj]).returncode: + return out + if step("link", [tools["clang"], "-fuse-ld=lld", obj, "-L", lib_dir, "-l" + MOJO_RUNTIME_LIB, + "-Wl,-rpath," + lib_dir, "-o", exe, DUMP_C]).returncode: return out ret = step("run", [exe], cwd=d) @@ -519,29 +478,18 @@ def step(stage, cmd, cwd=ROOT): return out t0 = time.monotonic() - if gcov: - gcdas = sorted(glob.glob(os.path.join(d, "*.gcda"))) - if not gcdas: - out.error = f"no .gcda written for {path} (binary did not exit normally?)" - return out - if step("collect", [tools["llvm-cov"], "gcov", "-i", *gcdas], cwd=d).returncode: - return out - for g in glob.glob(os.path.join(d, "*.gcov")): - with open(g) as f: - merge_coverage(out.coverage, parse_gcov_intermediate(f.read())) - else: - if not os.path.exists(dump): - out.error = f"no counter dump written for {path} (binary did not exit normally?)" - return out - with open(sites_path) as f: - sites = [tuple(s) for s in json.load(f)] - with open(dump, "rb") as f: - counts = counts_from_dump(sites, f.read()) - per_file = {} - for (p, n), c in counts.items(): - per_file.setdefault(p, {})[n] = c - merge_coverage(out.coverage, per_file) - out.stages["collect"] = out.stages.get("collect", 0.0) + time.monotonic() - t0 + if not os.path.exists(dump): + out.error = f"no counter dump written for {path} (binary did not exit normally?)" + return out + with open(sites_path) as f: + sites = [tuple(s) for s in json.load(f)] + with open(dump, "rb") as f: + counts = counts_from_dump(sites, f.read()) + per_file = {} + for (p, n), c in counts.items(): + per_file.setdefault(p, {})[n] = c + merge_coverage(out.coverage, per_file) + out.stages["collect"] = time.monotonic() - t0 if not out.coverage: out.error = f"no library lines collected for {path}" return out @@ -565,16 +513,12 @@ def main(): help="only test files whose path contains this substring") ap.add_argument("--missing", action="store_true", help="list the unexecuted line ranges per file") - ap.add_argument("--gcov", action="store_true", - help="use LLVM's gcov pass instead of the inline-aware" - " counters (blind to @always_inline bodies)") ap.add_argument("--opt-level", type=int, default=0, choices=(0, 1, 2, 3), help="mojo optimization level for the IR (default 0; higher" " levels merge and drop lines, use only for a file that" " misbehaves at -O0)") ap.add_argument("--llvm-bin", type=str, default=None, - help="directory holding llc, clang (and opt, llvm-cov for --gcov);" - " default: PATH") + help="directory holding llc and clang; default: PATH") ap.add_argument("--keep", action="store_true", help="keep the per-test IR, objects, binaries and counter data" " under .coverage/ (a failed file's are always kept)") @@ -596,13 +540,8 @@ def main(): if args.only and (args.check_baseline or args.update_baseline): sys.exit("ERROR: --only measures part of the suite; its total cannot" " be compared with or written to the baseline") - if args.gcov and (args.check_baseline or args.update_baseline): - sys.exit("ERROR: --gcov instruments a different (smaller) set of lines than" - " the default counters, so its total is not the baseline's." - " The baseline tracks the default mode.") - names = ("llc", "clang") + (("opt", "llvm-cov") if args.gcov else ()) - tools = find_tools(names, args.llvm_bin) + tools = find_tools(("llc", "clang"), args.llvm_bin) lib_dir = mojo_lib_dir() normal, _cfail = run_test.collect_files() if args.only: @@ -624,8 +563,7 @@ def main(): sys.exit("ERROR: mojo precompile failed (output above); try --no-pkg") include_dir = os.path.relpath(run_test.CACHE_DIR, ROOT) flags = run_test.mojo_flags(include_dir) - mode = "gcov" if args.gcov else "inline-aware counters" - print(f"{ver}; {llvm_ver} ({os.path.dirname(tools['llc'])}); {mode}, -O{args.opt_level};" + print(f"{ver}; {llvm_ver} ({os.path.dirname(tools['llc'])}); -O{args.opt_level};" f" {len(normal)} test files, -j{args.j}") for path, reason in skipped: print(f"skipped {path}: {reason}") @@ -635,8 +573,7 @@ def main(): results = run_test.load_results() outcomes = [] with ThreadPoolExecutor(max_workers=args.j) as pool: - futs = [pool.submit(cover_one, p, flags, tools, lib_dir, args.opt_level, - args.gcov, args.keep) + futs = [pool.submit(cover_one, p, flags, tools, lib_dir, args.opt_level, args.keep) for p in run_test.order_files(normal, results)] for fut in as_completed(futs): o = fut.result() @@ -658,9 +595,8 @@ def main(): failed = [o.path for o in outcomes if not o.ok] print() - print(f"Line coverage of emberregex/ over the suite ({mode}; runtime code only," - " comptime-executed lines are not instrumentable" - + ("; @always_inline bodies are not counted" if args.gcov else "") + ")") + print("Line coverage of emberregex/ over the suite (runtime code only," + " comptime-executed lines are not instrumentable)") if failed: print(f" WARNING: {len(failed)} of {len(outcomes)} test files failed" " (listed below); their coverage is missing from these numbers") diff --git a/tools/test_run_coverage.py b/tools/test_run_coverage.py index 952645c..28180a5 100644 --- a/tools/test_run_coverage.py +++ b/tools/test_run_coverage.py @@ -198,18 +198,6 @@ def test_counts_from_dump_sums_sites_on_one_line(self): self.assertEqual(rc.counts_from_dump(sites, blob), {("a.mojo", 1): 12, ("a.mojo", 2): 0}) -class TestGcovParse(unittest.TestCase): - def test_lcount_per_file_sums_repeats(self): - text = ( - "file:emberregex/a.mojo\nfunction:3,1,f\nlcount:3,1\nlcount:4,0\n" - "branch:4,taken\nlcount:3,2\nfile:/x/t.mojo\nlcount:9,1\n" - ) - self.assertEqual( - rc.parse_gcov_intermediate(text), - {"emberregex/a.mojo": {3: 3, 4: 0}, "/x/t.mojo": {9: 1}}, - ) - - class TestAggregation(unittest.TestCase): def test_library_path(self): self.assertEqual(rc.library_path("emberregex/engine.mojo"), "emberregex/engine.mojo") From 8deefc880f50e1c832bf89d79437a6ea89e9a2fc Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:02:13 -0600 Subject: [PATCH 11/65] cleanup: drop per-site build_bitmap in _parse_escape (parse() builds them all) Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/parser.mojo | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/emberregex/parser.mojo b/emberregex/parser.mojo index d830aef..2aa328a 100644 --- a/emberregex/parser.mojo +++ b/emberregex/parser.mojo @@ -117,7 +117,8 @@ struct Parser[origin: Origin](Movable): # top level there is no group to close). '|' is always eaten. assert self._peek() == CHAR_RPAREN, "parse stopped early" raise _regex_error("Unmatched ')'", self.pos) - # Build bitmaps for all charsets + # Build bitmaps for all charsets (the only place the parser's + # pooled charsets get one) for i in range(len(self.ast.charsets)): self.ast.charsets[i].build_bitmap() # Store inline flags on the AST so callers can access them @@ -802,7 +803,6 @@ struct Parser[origin: Origin](Movable): cs = CharSet.whitespace() if ch <= CHAR_Z_UPPER: # the uppercase forms cs.negate() - cs.build_bitmap() var cs_idx = self.ast.add_charset(cs^) return self.ast.add_node(ASTNode.char_class(cs_idx)) @@ -949,8 +949,8 @@ struct Parser[origin: Origin](Movable): # interpreter copies the aggregate across every call — over # the 684 ranges of `\p{L}` (934 of `\p{Word}`) that is a # quadratic ~470k scalar copies for what is 684 appends. - # `add_range` does nothing else but clear `bitmap_valid`, - # which `build_bitmap` sets again below. + # `add_range` does nothing else but clear `bitmap_valid`, and + # `parse()` builds every charset's bitmap once at the end. var pcs = CharSet() for i in range(len(pranges) // 2): pcs.ranges.append( @@ -958,7 +958,6 @@ struct Parser[origin: Origin](Movable): UInt32(pranges[2 * i]), UInt32(pranges[2 * i + 1]) ) ) - pcs.build_bitmap() var pidx = self.ast.add_charset(pcs^) return self.ast.add_node(ASTNode.char_class(pidx)) From 128602290162e7cc035fe97fc6fd5da202474c17 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:02:22 -0600 Subject: [PATCH 12/65] cleanup: add explicit flags parameter to Regex Regex[pattern, flags: RegexFlags = RegexFlags()] compiles the flags as a leading inline group (after any (*UTF8) verbs) through Regex._pat, which is exactly `pattern` for the default flags, so default instantiations keep the same memoized NFA and backtracker symbols. Drops the unused RegexFlags.__and__ and fixes the README Flags section. Co-Authored-By: Claude Opus 5.5 (1M context) --- ARCHITECTURE.md | 5 ++- README.md | 25 ++++++++--- emberregex/engine.mojo | 97 +++++++++++++++++++++++++++++++----------- emberregex/flags.mojo | 3 -- test/test_flags.mojo | 21 ++++++++- 5 files changed, 113 insertions(+), 38 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3f28c34..2614605 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -2,8 +2,9 @@ Two public entry points, one shared front end: -- **`Regex[pattern]`** — a single pattern, matched with Python-style - leftmost-first semantics. +- **`Regex[pattern, flags]`** — a single pattern, matched with Python-style + leftmost-first semantics (`flags` defaults to none; it is compiled as a + leading inline group, `Regex._pat`). - **`RegexSet[patterns]`** — a multi-pattern database in the shape of Intel Hyperscan: scan once, report every pattern that matches and where. diff --git a/README.md b/README.md index 1c3cb79..36f47a3 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ mojo -I /path/to/emberregex your_file.mojo ## API Reference -`Regex[pattern]` takes the pattern as a compile-time string literal. All parsing and NFA construction happen during compilation. +`Regex[pattern, flags]` takes the pattern as a compile-time string literal and optional [`RegexFlags`](#flags). All parsing and NFA construction happen during compilation. ### Matching @@ -160,7 +160,9 @@ var parts = re.split("one, two; three four") ### Flags -Pass flags as a second parameter, or use inline flag syntax in the pattern: +Pass `RegexFlags` as a second parameter, or use inline flag syntax in the +pattern. The parameter is sugar for the inline form: `Regex[p, flags]` +compiles `(?flags)p` (after any leading `(*UTF8)` verb). ```mojo from emberregex import Regex, RegexFlags @@ -173,16 +175,27 @@ re.match("HELLO").matched # True var re2 = Regex["(?i)hello"]() re2.match("HeLLo").matched # True -# Multiline: ^ and $ match at \n boundaries -var re3 = Regex["(?m)^\\w+"]() -var lines = re3.findall("foo\nbar\nbaz") -# lines: ["foo", "bar", "baz"] +# Combined flags: `|` the values (same as inline `(?im)`) +comptime IM = RegexFlags(RegexFlags.IGNORECASE) | RegexFlags( + RegexFlags.MULTILINE +) +var re3 = Regex["^[a-z]+", IM]() +var lines = re3.findall("foo\nBAR\nbaz") +# lines: ["foo", "BAR", "baz"] # Dotall: . matches \n var re4 = Regex["(?s)a.b"]() re4.match("a\nb").matched # True ``` +| Flag | Inline | Effect | +| --- | --- | --- | +| `RegexFlags.IGNORECASE` | `(?i)` | case-insensitive matching | +| `RegexFlags.MULTILINE` | `(?m)` | `^` and `$` also match at `\n` boundaries | +| `RegexFlags.DOTALL` | `(?s)` | `.` also matches `\n` | +| `RegexFlags.VERBOSE` | `(?x)` | whitespace and `#` comments in the pattern are ignored | +| `RegexFlags.UNICODE` | `(?u)`, `(*UTF8)` | UTF-8 mode: `.` and classes match one codepoint (see below) | + Bare inline flag groups like `(?i)` must appear **before any pattern content** (Python's rule — `a(?i)b` is a compile error). To apply flags to part of a pattern, use a scoped group: `a(?i:b)` or `(?-i:...)`. diff --git a/emberregex/engine.mojo b/emberregex/engine.mojo index 6c31855..9f760f2 100644 --- a/emberregex/engine.mojo +++ b/emberregex/engine.mojo @@ -15,9 +15,12 @@ backtrack.mojo for what that does and does not flatten. from .constants import ( CHAR_BACKSLASH, + CHAR_LPAREN, CHAR_NEWLINE, CHAR_NINE, CHAR_ONE, + CHAR_RPAREN, + CHAR_STAR, CHAR_ZERO, ) from .parser import parse @@ -1179,10 +1182,50 @@ struct _LFWalk[num_slots: Int, span: Bool, origin: MutOrigin]( self.pike = pike -struct Regex[pattern: String](Copyable, Movable): +def _apply_flags(pattern: String, flags: RegexFlags) -> String: + """`pattern` with `flags` spelled as one leading inline group + (`(?imsxu)`), placed after any leading `(*UTF8)` verbs — the parser + only accepts those first (see `Parser._consume_verbs`).""" + var letters = String() + if flags.ignorecase(): + letters += "i" + if flags.multiline(): + letters += "m" + if flags.dotall(): + letters += "s" + if flags.verbose(): + letters += "x" + if flags.unicode(): + letters += "u" + var b = pattern.as_bytes() + var pos = 0 + while ( + pos + 2 < len(b) and b[pos] == CHAR_LPAREN and b[pos + 1] == CHAR_STAR + ): + var close = pos + 2 + while close < len(b) and b[close] != CHAR_RPAREN: + close += 1 + if close >= len(b): + break + pos = close + 1 + return ( + String(unsafe_from_utf8=b[:pos]) + + "(?" + + letters + + ")" + + String(unsafe_from_utf8=b[pos:]) + ) + + +struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( + Copyable, Movable +): """A compile-time regex where parsing and NFA construction happen during compilation. + `flags` is the same as writing them inline: `Regex["ab", RegexFlags( + RegexFlags.IGNORECASE)]` compiles `(?i)ab`. + The backtracking engine is specialized per-NFA-state via comptime parameters. Each NFA state becomes a distinct function instantiation that keeps only the branch for its own kind, eliminating runtime dispatch on @@ -1190,7 +1233,15 @@ struct Regex[pattern: String](Copyable, Movable): recurse — see backtrack.mojo. """ - comptime nfa = _build_static_nfa(Self.pattern) + # The pattern every lane compiles: exactly `pattern` for the default + # flags, so those instantiations share the memoized NFA and the + # backtracker's symbol names with the flag-free spelling. The ternary + # is load-bearing: a String RETURNED by a comptime call is a different + # interpreter value (and symbol mangling) even when equal. + comptime _pat = Self.pattern if Self.flags.value == 0 else _apply_flags( + Self.pattern, Self.flags + ) + comptime nfa = _build_static_nfa(Self._pat) # One Tarjan pass and one depth plan per pattern: the selection # predicates below used to each call `split_cycle_flags(Self.nfa)` # inside their own bodies, and calls made inside interpreted bodies @@ -1713,7 +1764,7 @@ struct Regex[pattern: String](Copyable, Movable): left of it on return (negative after a -2). """ var end = _sbt_try_match[ - pattern=Self.pattern, + pattern=Self._pat, state_idx=Self._start, num_slots=Self._num_slots, anchored_end=False, @@ -2033,7 +2084,7 @@ struct Regex[pattern: String](Copyable, Movable): try: var memo = List[UInt64]() var got = _sbt_run[ - pattern=Self.pattern, + pattern=Self._pat, state_idx=Self._start, num_slots=Self._num_slots, anchored_end=True, @@ -2230,7 +2281,7 @@ struct Regex[pattern: String](Copyable, Movable): # alternatives that prefer a shorter match (e.g. `(a|ab)` # on "ab") can't mask a valid full match. var end = _sbt_run[ - pattern=Self.pattern, + pattern=Self._pat, state_idx=Self._start, num_slots=Self._num_slots, anchored_end=True, @@ -2336,9 +2387,7 @@ struct Regex[pattern: String](Copyable, Movable): end=self._lf_end_at( input_bytes, pos, match_end ), - slots=Array[Int, Self._num_slots]( - fill=-1 - ), + slots=Array[Int, Self._num_slots](fill=-1), ) var nl = simd_find_byte(input_bytes, CHAR_NEWLINE, pos) if nl < 0: @@ -2363,9 +2412,7 @@ struct Regex[pattern: String](Copyable, Movable): end=self._lf_end_at( input_bytes, pos, match_end ), - slots=Array[Int, Self._num_slots]( - fill=-1 - ), + slots=Array[Int, Self._num_slots](fill=-1), ) pos = _scan_bump[Self._is_unicode](input_bytes, pos) else: @@ -2379,9 +2426,7 @@ struct Regex[pattern: String](Copyable, Movable): end=self._lf_end_at( input_bytes, range[0], range[1] ), - slots=Array[Int, Self._num_slots]( - fill=-1 - ), + slots=Array[Int, Self._num_slots](fill=-1), ) return MatchResult[Self._num_slots].no_match() return MatchResult[Self._num_slots].no_match() @@ -2425,7 +2470,7 @@ struct Regex[pattern: String](Copyable, Movable): var sbt_memo = List[UInt64]() var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() var end = _sbt_run[ - pattern=Self.pattern, + pattern=Self._pat, state_idx=Self._start, num_slots=Self._num_slots, ]( @@ -2471,7 +2516,7 @@ struct Regex[pattern: String](Copyable, Movable): pos = self._next_candidate_pos(input, input_len, pos) var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() var end = _sbt_run[ - pattern=Self.pattern, + pattern=Self._pat, state_idx=Self._start, num_slots=Self._num_slots, ]( @@ -2527,7 +2572,7 @@ struct Regex[pattern: String](Copyable, Movable): continue var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() var end = _sbt_run[ - pattern=Self.pattern, + pattern=Self._pat, state_idx=entry_state, num_slots=Self._num_slots, ]( @@ -2942,7 +2987,7 @@ struct Regex[pattern: String](Copyable, Movable): comptime if Self._strategy.start_anchor == AnchorKind.BOL: var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() var end = _sbt_run[ - pattern=Self.pattern, + pattern=Self._pat, state_idx=Self._start, num_slots=Self._num_slots, ]( @@ -2964,7 +3009,7 @@ struct Regex[pattern: String](Copyable, Movable): while pos <= input_len: var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() var end = _sbt_run[ - pattern=Self.pattern, + pattern=Self._pat, state_idx=Self._start, num_slots=Self._num_slots, ]( @@ -3015,7 +3060,7 @@ struct Regex[pattern: String](Copyable, Movable): ) var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() var end = _sbt_run[ - pattern=Self.pattern, + pattern=Self._pat, state_idx=Self._start, num_slots=Self._num_slots, ]( @@ -3114,7 +3159,7 @@ struct Regex[pattern: String](Copyable, Movable): comptime if Self._strategy.start_anchor == AnchorKind.BOL: var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() var end = _sbt_run[ - pattern=Self.pattern, + pattern=Self._pat, state_idx=Self._start, num_slots=Self._num_slots, ]( @@ -3140,7 +3185,7 @@ struct Regex[pattern: String](Copyable, Movable): while pos <= input_len: var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() var end = _sbt_run[ - pattern=Self.pattern, + pattern=Self._pat, state_idx=Self._start, num_slots=Self._num_slots, ]( @@ -3195,7 +3240,7 @@ struct Regex[pattern: String](Copyable, Movable): ) var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() var end = _sbt_run[ - pattern=Self.pattern, + pattern=Self._pat, state_idx=Self._start, num_slots=Self._num_slots, ]( @@ -3427,7 +3472,7 @@ struct Regex[pattern: String](Copyable, Movable): pos = self._next_candidate_pos(input_bytes, input_len, pos) var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() var end = _sbt_run[ - pattern=Self.pattern, + pattern=Self._pat, state_idx=Self._start, num_slots=Self._num_slots, ]( @@ -3609,7 +3654,7 @@ struct Regex[pattern: String](Copyable, Movable): pos = self._next_candidate_pos(input_bytes, input_len, pos) var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() var end = _sbt_run[ - pattern=Self.pattern, + pattern=Self._pat, state_idx=Self._start, num_slots=Self._num_slots, ]( @@ -3832,7 +3877,7 @@ struct Regex[pattern: String](Copyable, Movable): var sbt_memo = List[UInt64]() var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() var end = _sbt_run[ - pattern=Self.pattern, + pattern=Self._pat, state_idx=Self._start, num_slots=Self._num_slots, ]( diff --git a/emberregex/flags.mojo b/emberregex/flags.mojo index 118a260..eb8e401 100644 --- a/emberregex/flags.mojo +++ b/emberregex/flags.mojo @@ -22,9 +22,6 @@ struct RegexFlags(ImplicitlyCopyable, Movable): def __or__(self, other: Self) -> Self: return RegexFlags(self.value | other.value) - def __and__(self, other: Self) -> Self: - return RegexFlags(self.value & other.value) - def has(self, flag: Int) -> Bool: return (self.value & flag) != 0 diff --git a/test/test_flags.mojo b/test/test_flags.mojo index 209bc1f..58b0468 100644 --- a/test/test_flags.mojo +++ b/test/test_flags.mojo @@ -1,6 +1,6 @@ """Tests for inline flags: (?i) ignorecase, (?m) multiline, (?s) dotall.""" -from emberregex import Regex +from emberregex import Regex, RegexFlags from std.testing import assert_true, assert_false, assert_equal, TestSuite @@ -110,6 +110,25 @@ def test_combined_ignorecase_multiline() raises: assert_true(re.search("foo\nHELLO").matched) +def test_explicit_flags_param() raises: + # Regex[p, flags] compiles `(?flags)p` (after any leading verbs) and + # the default flags leave the pattern untouched. + comptime I = RegexFlags(RegexFlags.IGNORECASE) + comptime IM = I | RegexFlags(RegexFlags.MULTILINE) + comptime assert Regex["(?i)hello"]._pat == "(?i)hello" + comptime assert Regex["hello", I]._pat == "(?i)hello" + comptime assert Regex["(*UTF8)a", IM]._pat == "(*UTF8)(?im)a" + var re = Regex["hello", I]() + var inline = Regex["(?i)hello"]() + for s in ["hElLo", "HELLO", "hell"]: + assert_equal(re.match(s).matched, inline.match(s).matched) + var re2 = Regex["^hello", IM]() + var inline2 = Regex["(?im)^hello"]() + var r = re2.search("foo\nHELLO") + assert_true(r.matched) + assert_equal(r.span(), inline2.search("foo\nHELLO").span()) + + def test_combined_all_three() raises: var re = Regex["(?ims)^hello.world$"]() assert_true(re.search("HELLO\nWORLD").matched) From 21bdffd630b452e2851a98d902bf64745de6055b Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:02:40 -0600 Subject: [PATCH 13/65] cleanup: _utf8_trie_fragment builds its own root index list Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/nfa.mojo | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/emberregex/nfa.mojo b/emberregex/nfa.mojo index b9a956c..5c2d1e2 100644 --- a/emberregex/nfa.mojo +++ b/emberregex/nfa.mojo @@ -600,10 +600,7 @@ def _utf8_class_fragment(mut nfa: NFA, ranges: List[Int]) raises -> NFAFragment: frag.add_out(st, 1) return frag^ - var all_idx = List[Int]() - for i in range(tbl.count): - all_idx.append(i) - return _utf8_trie_fragment(nfa, tbl.words, all_idx, 0) + return _utf8_trie_fragment(nfa, tbl.words, tbl.count) # Field width of the packed trie records and bucket descriptors built by @@ -621,10 +618,10 @@ comptime _TRIE_ID_LIMIT = 1 << _TRIE_FIELD_BITS def _utf8_trie_fragment( mut nfa: NFA, seq_words: List[Int], - idxs: List[Int], - pos: Int, + count: Int, ) raises -> NFAFragment: - """Prefix-factored alternation over byte-range sequences. + """Prefix-factored alternation over the `count` byte-range sequences + packed in `seq_words`. Emitting one independent chain per sequence is correct but ruinous for the big Unicode classes: `\\p{L}` is 805 sequences, so the naive @@ -650,7 +647,7 @@ def _utf8_trie_fragment( bucket the same thing (a first-seen scan plus a stable counting sort backs the fast path up, so unsorted inputs still factor correctly). """ - if len(idxs) >= _TRIE_ID_LIMIT: + if count >= _TRIE_ID_LIMIT: raise Error("utf8 trie: too many sequences") # Local state records, one packed Int each; local ids materialize at @@ -665,11 +662,14 @@ def _utf8_trie_fragment( # Worklist of subtrees: member indices, byte position, and the local # charset state whose out1 the subtree start patches (-1 = root). + var all_idx = List[Int]() + for i in range(count): + all_idx.append(i) var task_idxs = List[List[Int]]() var task_pos = List[Int]() var task_patch = List[Int]() - task_idxs.append(idxs.copy()) - task_pos.append(pos) + task_idxs.append(all_idx^) + task_pos.append(0) task_patch.append(-1) var t = 0 From 54fd26ae53b8da034846e79bb243c644c7e9e28b Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:03:15 -0600 Subject: [PATCH 14/65] cleanup: one ascii_to_lower helper in constants.mojo Replaces _sbt_to_lower, _bt_to_lower and nfa.mojo's _to_lower (same ASCII A-Z rule; generic over the scalar type so the byte walkers and the UInt32 codepoint builder share it). Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/backtrack.mojo | 20 ++++---------------- emberregex/constants.mojo | 9 +++++++++ emberregex/executor.mojo | 17 +++-------------- emberregex/nfa.mojo | 17 ++++++++--------- 4 files changed, 24 insertions(+), 39 deletions(-) diff --git a/emberregex/backtrack.mojo b/emberregex/backtrack.mojo index 1fc10c2..6c3b12e 100644 --- a/emberregex/backtrack.mojo +++ b/emberregex/backtrack.mojo @@ -42,12 +42,7 @@ from std.ffi import external_call from std.sys.info import CompilationTarget from std.sys.intrinsics import llvm_intrinsic -from .constants import ( - CHAR_A_UPPER, - CHAR_NEWLINE, - CHAR_Z_UPPER, - is_word_byte, -) +from .constants import CHAR_NEWLINE, ascii_to_lower, is_word_byte from .nfa import ( _build_static_nfa, split_cycle_flags, @@ -66,13 +61,6 @@ from .simd_kernels import ( ) -@always_inline -def _sbt_to_lower(ch: Byte) -> Byte: - if ch >= CHAR_A_UPPER and ch <= CHAR_Z_UPPER: - return ch + 32 - return ch - - @always_inline def _sbt_bitmap_check( bitmap: SIMD[DType.uint8, BITMAP_WIDTH], negated: Bool, ch: UInt32 @@ -2190,9 +2178,9 @@ def _sbt_try_match[ return -1 comptime if state.icase: for i in range(ref_len): - if _sbt_to_lower(input.unsafe_get(gs + i)) != _sbt_to_lower( - input.unsafe_get(pos + i) - ): + if ascii_to_lower( + input.unsafe_get(gs + i) + ) != ascii_to_lower(input.unsafe_get(pos + i)): return -1 else: for i in range(ref_len): diff --git a/emberregex/constants.mojo b/emberregex/constants.mojo index 238dbd5..90ce74b 100644 --- a/emberregex/constants.mojo +++ b/emberregex/constants.mojo @@ -82,3 +82,12 @@ def is_word_byte(ch: Byte) -> Bool: or (ch >= CHAR_ZERO and ch <= CHAR_NINE) or ch == CHAR_UNDERSCORE ) + + +@always_inline +def ascii_to_lower[dt: DType, //](ch: Scalar[dt]) -> Scalar[dt]: + """ASCII `A-Z` to `a-z`; every other value (bytes or codepoints) is + returned unchanged.""" + if ch >= CHAR_A_UPPER.cast[dt]() and ch <= CHAR_Z_UPPER.cast[dt](): + return ch + 32 + return ch diff --git a/emberregex/executor.mojo b/emberregex/executor.mojo index 9dd9acf..9020030 100644 --- a/emberregex/executor.mojo +++ b/emberregex/executor.mojo @@ -10,12 +10,7 @@ per-state heap allocations. SAVE states use in-place modification with restore-on-return to eliminate slot copying. """ -from .constants import ( - CHAR_A_UPPER, - CHAR_NEWLINE, - CHAR_Z_UPPER, - is_word_byte, -) +from .constants import CHAR_NEWLINE, ascii_to_lower, is_word_byte from .nfa import NFA, NFAStateKind from .ast import AnchorKind from .result import MatchResult @@ -769,9 +764,9 @@ def _heapbt_core[ var same = True if state.icase: for i in range(n): - if _bt_to_lower( + if ascii_to_lower( input.unsafe_get(gs + i) - ) != _bt_to_lower(input.unsafe_get(pos + i)): + ) != ascii_to_lower(input.unsafe_get(pos + i)): same = False break else: @@ -814,9 +809,3 @@ def _bt_check_anchor[ var right = pos < input_len and is_word_byte(input.unsafe_get(pos)) return (left != right) == (anchor_type == AnchorKind.WORD_BOUNDARY) return False - - -def _bt_to_lower(ch: Byte) -> Byte: - if ch >= CHAR_A_UPPER and ch <= CHAR_Z_UPPER: - return ch + 32 - return ch diff --git a/emberregex/nfa.mojo b/emberregex/nfa.mojo index c898628..57cd5db 100644 --- a/emberregex/nfa.mojo +++ b/emberregex/nfa.mojo @@ -7,7 +7,13 @@ a list of dangling output arrows (patch list). from std.math import max, min -from .constants import CHAR_A_LOWER, CHAR_A_UPPER, CHAR_Z_LOWER, CHAR_Z_UPPER +from .constants import ( + CHAR_A_LOWER, + CHAR_A_UPPER, + CHAR_Z_LOWER, + CHAR_Z_UPPER, + ascii_to_lower, +) from .ast import AST, ASTNode, ASTNodeKind, AnchorKind from .charset import BITMAP_WIDTH, CharSet, CharRange from .utf8 import UTF8_SEQ_LEN_SHIFT, UTF8_SEQ_WORDS, utf8_seq_table @@ -975,7 +981,7 @@ def _build_fragment( one.append(Int(ch)) return _utf8_class_fragment(nfa, one) if flags.ignorecase(): - var lo = _to_lower(ch) + var lo = ascii_to_lower(ch) var up = _to_upper(ch) if lo != up: var cs = CharSet() @@ -1493,13 +1499,6 @@ def _build_repetition( return frag^ -def _to_lower(ch: UInt32) -> UInt32: - """Convert ASCII uppercase to lowercase.""" - if ch >= UInt32(CHAR_A_UPPER) and ch <= UInt32(CHAR_Z_UPPER): - return ch + 32 - return ch - - def _to_upper(ch: UInt32) -> UInt32: """Convert ASCII lowercase to uppercase.""" if ch >= UInt32(CHAR_A_LOWER) and ch <= UInt32(CHAR_Z_LOWER): From 8c52cbded5745f68182bcec4b15fb74def4e6a43 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:04:30 -0600 Subject: [PATCH 15/65] cleanup: one parametric anchor-continuation walk for the three DFA-lane guards Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/static_dfa.mojo | 151 +++++++++++++------------------------ 1 file changed, 52 insertions(+), 99 deletions(-) diff --git a/emberregex/static_dfa.mojo b/emberregex/static_dfa.mojo index e31e0de..c4299bb 100644 --- a/emberregex/static_dfa.mojo +++ b/emberregex/static_dfa.mojo @@ -551,44 +551,6 @@ def _wb_normalize(fl: Int) -> Int: return fl -def _wb_cont_reaches_bol(nfa: NFA) -> Bool: - """Comptime: does some word anchor's epsilon continuation reach a BOL - kind (`\\b^`, `(?m)\\b^x`)? The DFA lanes expand a word anchor's - continuation when the anchor resolves, without the position context - a BOL kind needs, so such patterns stay off them (the mirror of - `_eol_continuation_crosses_anchor`). The walk follows every anchor - conservatively.""" - var num_states = len(nfa.states) - for i in range(num_states): - if nfa.states[i].kind != NFAStateKind.ANCHOR: - continue - var at = nfa.states[i].anchor_type - if ( - at != AnchorKind.WORD_BOUNDARY - and at != AnchorKind.NOT_WORD_BOUNDARY - ): - continue - var visited = List[Bool](length=num_states, fill=False) - var stack: List[Int] = [nfa.states[i].out1] - while len(stack) > 0: - var s = stack.pop() - if s < 0 or s >= num_states or visited[s]: - continue - visited[s] = True - var kind = nfa.states[s].kind - if kind == NFAStateKind.SPLIT: - stack.append(nfa.states[s].out1) - stack.append(nfa.states[s].out2) - elif kind == NFAStateKind.SAVE: - stack.append(nfa.states[s].out1) - elif kind == NFAStateKind.ANCHOR: - var at2 = nfa.states[s].anchor_type - if at2 == AnchorKind.BOL or at2 == AnchorKind.BOL_MULTILINE: - return True - stack.append(nfa.states[s].out1) - return False - - struct EagerDFA(Copyable, Movable): """Comptime-computed DFA: flat transition table + per-state flags. @@ -675,22 +637,20 @@ struct EagerDFA(Copyable, Movable): self.region_land = List[Int]() -def _eol_ml_continuation_consumes(nfa: NFA) -> Bool: - """Comptime: does any EOL_MULTILINE anchor's continuation consume? - - The DFA lanes keep EOL anchors unresolved in state sets and resolve - them via per-state flags, so a continuation that must consume more - input (e.g. `(?m)a$\\nb`) is unreachable there — the DFA silently - under-reports. Such patterns must stay off the DFA lanes. Strict EOL - needs no such guard: it holds only at end of input, where a - consuming continuation is provably dead. The walk follows any - anchor conservatively (assume it could hold). - """ +def _anchor_cont_hits[ + seed_mask: Int, hit_anchor_mask: Int, hit_consuming: Bool +](nfa: NFA) -> Bool: + """Comptime: does the epsilon continuation (`out1`) of some anchor whose + kind is in `seed_mask` reach an anchor whose kind is in + `hit_anchor_mask`, or, with `hit_consuming`, a consuming state (CHAR, + CHARSET, ANY, BACKREF)? Masks are `1 << AnchorKind`. The walk follows + SPLIT, SAVE and every other anchor (conservatively assuming it holds); + anything else ends it.""" var num_states = len(nfa.states) for i in range(num_states): if nfa.states[i].kind != NFAStateKind.ANCHOR: continue - if nfa.states[i].anchor_type != AnchorKind.EOL_MULTILINE: + if (seed_mask >> nfa.states[i].anchor_type) & 1 == 0: continue var visited = List[Bool](length=num_states, fill=False) var stack: List[Int] = [nfa.states[i].out1] @@ -700,64 +660,57 @@ def _eol_ml_continuation_consumes(nfa: NFA) -> Bool: continue visited[s] = True var kind = nfa.states[s].kind - if ( + if kind == NFAStateKind.SPLIT: + stack.append(nfa.states[s].out1) + stack.append(nfa.states[s].out2) + elif kind == NFAStateKind.SAVE: + stack.append(nfa.states[s].out1) + elif kind == NFAStateKind.ANCHOR: + if (hit_anchor_mask >> nfa.states[s].anchor_type) & 1 != 0: + return True + stack.append(nfa.states[s].out1) + elif hit_consuming and ( kind == NFAStateKind.CHAR or kind == NFAStateKind.CHARSET or kind == NFAStateKind.ANY or kind == NFAStateKind.BACKREF ): return True - if kind == NFAStateKind.SPLIT: - stack.append(nfa.states[s].out1) - stack.append(nfa.states[s].out2) - elif kind == NFAStateKind.SAVE or kind == NFAStateKind.ANCHOR: - stack.append(nfa.states[s].out1) return False -def _eol_continuation_crosses_anchor(nfa: NFA) -> Bool: - """Comptime: does any EOL anchor's continuation reach an anchor whose - truth is NOT implied by the EOL that precedes it? - - The DFA lanes resolve EOL anchors with per-state flag bytes, which - carry one bit of context ("we are at a '\\n'" / "we are at the end"). - A nested EOL anchor is fine — `_reaches_match` follows the kinds that - hold in the same context, which is what makes `ab$$` work. A BOL kind - or a word boundary is not: whether it holds depends on the *preceding* - byte, which the flag cannot express, so the walk would have to guess. - Such patterns stay off these lanes rather than guess (the same - treatment `_eol_ml_continuation_consumes` gives consuming - continuations). - """ - var num_states = len(nfa.states) - for i in range(num_states): - if nfa.states[i].kind != NFAStateKind.ANCHOR: - continue - var at = nfa.states[i].anchor_type - if at != AnchorKind.EOL and at != AnchorKind.EOL_MULTILINE: - continue - var visited = List[Bool](length=num_states, fill=False) - var stack: List[Int] = [nfa.states[i].out1] - while len(stack) > 0: - var s = stack.pop() - if s < 0 or s >= num_states or visited[s]: - continue - visited[s] = True - var kind = nfa.states[s].kind - if kind == NFAStateKind.SPLIT: - stack.append(nfa.states[s].out1) - stack.append(nfa.states[s].out2) - elif kind == NFAStateKind.SAVE: - stack.append(nfa.states[s].out1) - elif kind == NFAStateKind.ANCHOR: - var at2 = nfa.states[s].anchor_type - if at2 == AnchorKind.EOL or at2 == AnchorKind.EOL_MULTILINE: - stack.append(nfa.states[s].out1) - else: - return True - # consuming states end the walk; the consuming case has its - # own guard (_eol_ml_continuation_consumes) - return False +comptime _EOL_KINDS = (1 << AnchorKind.EOL) | (1 << AnchorKind.EOL_MULTILINE) + +comptime _wb_cont_reaches_bol = _anchor_cont_hits[ + (1 << AnchorKind.WORD_BOUNDARY) | (1 << AnchorKind.NOT_WORD_BOUNDARY), + (1 << AnchorKind.BOL) | (1 << AnchorKind.BOL_MULTILINE), + False, +] +"""Does some word anchor's continuation reach a BOL kind (`\\b^`, +`(?m)\\b^x`)? The DFA lanes expand a word anchor's continuation when the +anchor resolves, without the position context a BOL kind needs, so such +patterns stay off them.""" + +comptime _eol_ml_continuation_consumes = _anchor_cont_hits[ + 1 << AnchorKind.EOL_MULTILINE, 0, True +] +"""Does any EOL_MULTILINE anchor's continuation consume? The DFA lanes keep +EOL anchors unresolved in state sets and resolve them via per-state flags, +so a continuation that must consume more input (`(?m)a$\\nb`) is +unreachable there and the DFA would silently under-report. Strict EOL +needs no such guard: it holds only at end of input, where a consuming +continuation is provably dead.""" + +comptime _eol_continuation_crosses_anchor = _anchor_cont_hits[ + _EOL_KINDS, ~_EOL_KINDS, False +] +"""Does any EOL anchor's continuation reach an anchor whose truth is NOT +implied by the EOL before it? The flag bytes carry one bit of context ("at +a '\\n'" / "at the end"): a nested EOL is fine (`_reaches_match` follows +it, which is what makes `ab$$` work), but a BOL kind or word boundary +depends on the PRECEDING byte, so such patterns stay off these lanes +rather than guess. (Consuming continuations are +`_eol_ml_continuation_consumes`'s.)""" # Folded into a state's hash when its look-behind class is "word". From 0d43368d17709175f3a06b85997f88271e0785f5 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:04:46 -0600 Subject: [PATCH 16/65] cleanup: merge LOOKAHEAD/LOOKBEHIND arms in the backtracker and Pike VM Only the sub-match differs; the negated / keep-restore tail is shared. The backtracker picks the sub-match with a comptime if, so each state still elaborates exactly one arm. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/backtrack.mojo | 91 +++++++++++---------------------------- emberregex/executor.mojo | 65 ++++++++++++---------------- 2 files changed, 52 insertions(+), 104 deletions(-) diff --git a/emberregex/backtrack.mojo b/emberregex/backtrack.mojo index 6c3b12e..8f3bc0e 100644 --- a/emberregex/backtrack.mojo +++ b/emberregex/backtrack.mojo @@ -2045,83 +2045,39 @@ def _sbt_try_match[ ](input, pos, slots, budget, memo_addr, stack_floor, end_at) return -1 - elif kind == NFAStateKind.LOOKAHEAD: + elif ( + kind == NFAStateKind.LOOKAHEAD or kind == NFAStateKind.LOOKBEHIND + ): + var matched = False var sub_slots = slots.copy() - var sub_result = _sbt_try_match[ - pattern=pattern, - state_idx=state.sub_start, - num_slots=num_slots, - anchored_end=False, - memo_on=memo_on, - ](input, pos, sub_slots, budget, memo_addr, stack_floor, end_at) - var matched = sub_result >= 0 - comptime if state.negated: - if not matched: - return _sbt_try_match[ - pattern=pattern, - state_idx=state.out1, - num_slots=num_slots, - anchored_end=anchored_end, - memo_on=memo_on, - ]( - input, - pos, - slots, - budget, - memo_addr, - stack_floor, - end_at, - ) - return -1 + comptime if kind == NFAStateKind.LOOKAHEAD: + var sub_result = _sbt_try_match[ + pattern=pattern, + state_idx=state.sub_start, + num_slots=num_slots, + anchored_end=False, + memo_on=memo_on, + ](input, pos, sub_slots, budget, memo_addr, stack_floor, end_at) + matched = sub_result >= 0 else: - if matched: - # A successful positive assertion KEEPS its capture - # writes (Python/PCRE/Perl/Ruby/JS all agree); restore - # them if the continuation fails so outer backtracking - # cannot leak them into other attempts. - var saved_slots = slots.copy() - slots = sub_slots^ - var cont = _sbt_try_match[ + comptime lb_len = state.lookbehind_len + if pos >= lb_len: + var sub_result = _sbt_try_match[ pattern=pattern, - state_idx=state.out1, + state_idx=state.sub_start, num_slots=num_slots, - anchored_end=anchored_end, + anchored_end=False, memo_on=memo_on, ]( input, - pos, - slots, + pos - lb_len, + sub_slots, budget, memo_addr, stack_floor, end_at, ) - if cont < 0: - slots = saved_slots^ - return cont - return -1 - - elif kind == NFAStateKind.LOOKBEHIND: - comptime lb_len = state.lookbehind_len - var matched = False - var sub_slots = slots.copy() - if pos >= lb_len: - var sub_result = _sbt_try_match[ - pattern=pattern, - state_idx=state.sub_start, - num_slots=num_slots, - anchored_end=False, - memo_on=memo_on, - ]( - input, - pos - lb_len, - sub_slots, - budget, - memo_addr, - stack_floor, - end_at, - ) - matched = sub_result >= 0 and sub_result == pos + matched = sub_result >= 0 and sub_result == pos comptime if state.negated: if not matched: return _sbt_try_match[ @@ -2142,7 +2098,10 @@ def _sbt_try_match[ return -1 else: if matched: - # Same keep/restore rule as LOOKAHEAD above. + # A successful positive assertion KEEPS its capture + # writes (Python/PCRE/Perl/Ruby/JS all agree); restore + # them if the continuation fails so outer backtracking + # cannot leak them into other attempts. var saved_slots = slots.copy() slots = sub_slots^ var cont = _sbt_try_match[ diff --git a/emberregex/executor.mojo b/emberregex/executor.mojo index 9020030..451601c 100644 --- a/emberregex/executor.mojo +++ b/emberregex/executor.mojo @@ -432,14 +432,35 @@ struct PikeVM[num_slots: Int](Copyable): continue return - elif kind == NFAStateKind.LOOKAHEAD: + elif ( + kind == NFAStateKind.LOOKAHEAD + or kind == NFAStateKind.LOOKBEHIND + ): gen.unsafe_set(state_idx, gen_val) + var matched = False var sub_slots = slots.copy() - var match_end = _bt_try_match( - self.nfa, input, state.sub_start, pos, sub_slots - ) - if (match_end >= 0) != state.negated: - if match_end >= 0: + if kind == NFAStateKind.LOOKAHEAD: + matched = ( + _bt_try_match( + self.nfa, input, state.sub_start, pos, sub_slots + ) + >= 0 + ) + else: + var lb_len = state.lookbehind_len + if pos >= lb_len: + matched = ( + _bt_try_match( + self.nfa, + input, + state.sub_start, + pos - lb_len, + sub_slots, + ) + == pos + ) + if matched != state.negated: + if matched: # Successful POSITIVE assertion: the continuation # sees its capture writes; restore after the # subtree so sibling threads are unaffected @@ -463,38 +484,6 @@ struct PikeVM[num_slots: Int](Copyable): continue return - elif kind == NFAStateKind.LOOKBEHIND: - gen.unsafe_set(state_idx, gen_val) - var lb_len = state.lookbehind_len - var lb_matched = False - var lb_slots = slots.copy() - if pos >= lb_len: - var match_end = _bt_try_match( - self.nfa, input, state.sub_start, pos - lb_len, lb_slots - ) - lb_matched = match_end == pos - if lb_matched != state.negated: - if lb_matched: - # Same keep/restore rule as LOOKAHEAD above. - var saved_slots = slots.copy() - slots = lb_slots^ - self._add_state( - state_list, - slot_data, - gen, - gen_val, - state.out1, - slots, - input, - input_len, - pos, - ) - slots = saved_slots^ - return - state_idx = state.out1 - continue - return - else: # Consuming state (CHAR, CHARSET, ANY, MATCH) — commit to flat array gen.unsafe_set(state_idx, gen_val) From 027e6e514cd125d73e74a11cf7e0efdde442d770 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:05:32 -0600 Subject: [PATCH 17/65] cleanup: one Regex._bt helper for the 14 _sbt_run call sites Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/engine.mojo | 207 ++++++++--------------------------------- 1 file changed, 39 insertions(+), 168 deletions(-) diff --git a/emberregex/engine.mojo b/emberregex/engine.mojo index 9f760f2..733d81c 100644 --- a/emberregex/engine.mojo +++ b/emberregex/engine.mojo @@ -1713,6 +1713,28 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( flags=Self._EDFA_FLAGS, ](input, start) + @always_inline + def _bt[ + origin: Origin, + //, + state_idx: Int = Self._start, + anchored_end: Bool = False, + ]( + self, + input: Span[Byte, origin], + pos: Int, + mut slots: Array[Int, Self._num_slots], + mut memo: List[UInt64], + end_at: Int = -1, + ) raises -> Int: + """`_sbt_run` on this pattern with this Regex's cached stack range.""" + return _sbt_run[ + pattern=Self._pat, + state_idx=state_idx, + num_slots=Self._num_slots, + anchored_end=anchored_end, + ](input, pos, slots, memo, end_at, self._stack_lo, self._stack_hi) + @always_inline def _sbt_match_at[ origin: Origin, // @@ -2083,19 +2105,8 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( if pike[].sbt_ok: try: var memo = List[UInt64]() - var got = _sbt_run[ - pattern=Self._pat, - state_idx=Self._start, - num_slots=Self._num_slots, - anchored_end=True, - ]( - input, - start, - slots, - memo, - end_at=end, - stack_lo=self._stack_lo, - stack_hi=self._stack_hi, + var got = self._bt[anchored_end=True]( + input, start, slots, memo, end_at=end ) if got == end: return @@ -2280,18 +2291,8 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( # anchored_end: MATCH only accepts at end of input, so # alternatives that prefer a shorter match (e.g. `(a|ab)` # on "ab") can't mask a valid full match. - var end = _sbt_run[ - pattern=Self._pat, - state_idx=Self._start, - num_slots=Self._num_slots, - anchored_end=True, - ]( - input.as_bytes(), - 0, - slots, - sbt_memo, - stack_lo=self._stack_lo, - stack_hi=self._stack_hi, + var end = self._bt[anchored_end=True]( + input.as_bytes(), 0, slots, sbt_memo ) if end >= 0: return MatchResult[Self._num_slots]( @@ -2469,18 +2470,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( # other two branches own theirs. var sbt_memo = List[UInt64]() var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() - var end = _sbt_run[ - pattern=Self._pat, - state_idx=Self._start, - num_slots=Self._num_slots, - ]( - input_bytes, - 0, - slots, - sbt_memo, - stack_lo=self._stack_lo, - stack_hi=self._stack_hi, - ) + var end = self._bt(input_bytes, 0, slots, sbt_memo) if end >= 0: return MatchResult[Self._num_slots]( matched=True, @@ -2515,18 +2505,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( comptime if Self._strategy.first_byte_useful: pos = self._next_candidate_pos(input, input_len, pos) var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() - var end = _sbt_run[ - pattern=Self._pat, - state_idx=Self._start, - num_slots=Self._num_slots, - ]( - input, - pos, - slots, - sbt_memo, - stack_lo=self._stack_lo, - stack_hi=self._stack_hi, - ) + var end = self._bt(input, pos, slots, sbt_memo) if end >= 0: return MatchResult[Self._num_slots]( matched=True, @@ -2571,17 +2550,8 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( pos = nl + 1 continue var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() - var end = _sbt_run[ - pattern=Self._pat, - state_idx=entry_state, - num_slots=Self._num_slots, - ]( - input, - pos, - slots, - sbt_memo, - stack_lo=self._stack_lo, - stack_hi=self._stack_hi, + var end = self._bt[state_idx=entry_state]( + input, pos, slots, sbt_memo ) if end >= 0: return MatchResult[Self._num_slots]( @@ -2986,18 +2956,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( # BOL anchor: only position 0 comptime if Self._strategy.start_anchor == AnchorKind.BOL: var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() - var end = _sbt_run[ - pattern=Self._pat, - state_idx=Self._start, - num_slots=Self._num_slots, - ]( - input_bytes, - 0, - slots, - sbt_memo, - stack_lo=self._stack_lo, - stack_hi=self._stack_hi, - ) + var end = self._bt(input_bytes, 0, slots, sbt_memo) if end >= 0: self._findall_append(results, input, 0, end, slots) return results^ @@ -3008,18 +2967,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( var pos = 0 while pos <= input_len: var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() - var end = _sbt_run[ - pattern=Self._pat, - state_idx=Self._start, - num_slots=Self._num_slots, - ]( - input_bytes, - pos, - slots, - sbt_memo, - stack_lo=self._stack_lo, - stack_hi=self._stack_hi, - ) + var end = self._bt(input_bytes, pos, slots, sbt_memo) if end >= 0: self._findall_append(results, input, pos, end, slots) if end > pos: @@ -3059,18 +3007,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( input_bytes, input_len, pos ) var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() - var end = _sbt_run[ - pattern=Self._pat, - state_idx=Self._start, - num_slots=Self._num_slots, - ]( - input_bytes, - pos, - slots, - sbt_memo, - stack_lo=self._stack_lo, - stack_hi=self._stack_hi, - ) + var end = self._bt(input_bytes, pos, slots, sbt_memo) if end < 0: pos = _scan_bump[Self._is_unicode](input_bytes, pos) continue @@ -3158,18 +3095,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( # BOL anchor: only position 0 comptime if Self._strategy.start_anchor == AnchorKind.BOL: var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() - var end = _sbt_run[ - pattern=Self._pat, - state_idx=Self._start, - num_slots=Self._num_slots, - ]( - input_bytes, - 0, - slots, - sbt_memo, - stack_lo=self._stack_lo, - stack_hi=self._stack_hi, - ) + var end = self._bt(input_bytes, 0, slots, sbt_memo) if end >= 0: results.append( MatchResult[Self._num_slots]( @@ -3184,18 +3110,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( var pos = 0 while pos <= input_len: var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() - var end = _sbt_run[ - pattern=Self._pat, - state_idx=Self._start, - num_slots=Self._num_slots, - ]( - input_bytes, - pos, - slots, - sbt_memo, - stack_lo=self._stack_lo, - stack_hi=self._stack_hi, - ) + var end = self._bt(input_bytes, pos, slots, sbt_memo) if end >= 0: results.append( MatchResult[Self._num_slots]( @@ -3239,18 +3154,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( input_bytes, input_len, pos ) var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() - var end = _sbt_run[ - pattern=Self._pat, - state_idx=Self._start, - num_slots=Self._num_slots, - ]( - input_bytes, - pos, - slots, - sbt_memo, - stack_lo=self._stack_lo, - stack_hi=self._stack_hi, - ) + var end = self._bt(input_bytes, pos, slots, sbt_memo) if end < 0: pos = _scan_bump[Self._is_unicode](input_bytes, pos) continue @@ -3471,18 +3375,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( comptime if Self._strategy.first_byte_useful: pos = self._next_candidate_pos(input_bytes, input_len, pos) var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() - var end = _sbt_run[ - pattern=Self._pat, - state_idx=Self._start, - num_slots=Self._num_slots, - ]( - input_bytes, - pos, - slots, - sbt_memo, - stack_lo=self._stack_lo, - stack_hi=self._stack_hi, - ) + var end = self._bt(input_bytes, pos, slots, sbt_memo) if end < 0: pos = _scan_bump[Self._is_unicode](input_bytes, pos) continue @@ -3653,18 +3546,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( comptime if Self._strategy.first_byte_useful: pos = self._next_candidate_pos(input_bytes, input_len, pos) var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() - var end = _sbt_run[ - pattern=Self._pat, - state_idx=Self._start, - num_slots=Self._num_slots, - ]( - input_bytes, - pos, - slots, - sbt_memo, - stack_lo=self._stack_lo, - stack_hi=self._stack_hi, - ) + var end = self._bt(input_bytes, pos, slots, sbt_memo) if end < 0: pos = _scan_bump[Self._is_unicode](input_bytes, pos) continue @@ -3876,18 +3758,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( # and a destructor edge on a path that never uses it. var sbt_memo = List[UInt64]() var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() - var end = _sbt_run[ - pattern=Self._pat, - state_idx=Self._start, - num_slots=Self._num_slots, - ]( - input, - start, - slots, - sbt_memo, - stack_lo=self._stack_lo, - stack_hi=self._stack_hi, - ) + var end = self._bt(input, start, slots, sbt_memo) if end >= 0: return end except: From d9070e397c925096d5bbae2ca846ee492f6c1145 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:05:32 -0600 Subject: [PATCH 18/65] cleanup: fold edfa_walk_from/sheng_walk_from and the LF find_end wrappers into edfa_match_at/sheng_match_at Every caller passed the DFA's own start states; the LF wrappers passed lf.d's, so they were edfa_match_at/sheng_match_at over lf.d. Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 2 +- emberregex/engine.mojo | 14 ++--- emberregex/sheng.mojo | 74 +++++--------------------- emberregex/static_dfa.mojo | 88 ++++++++----------------------- emberregex/static_lfdfa.mojo | 58 -------------------- test/test_leftmost_first_dfa.mojo | 13 ++--- test/test_word_boundary_dfa.mojo | 6 ++- 7 files changed, 50 insertions(+), 205 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index cf851e7..441e642 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -67,7 +67,7 @@ Engine selection happens at compile time via `comptime if` branches: **Eager DFA** (`static_dfa.mojo`) — the default DFA engine when `can_use_dfa` is true and `group_count == 0` (with captures the same tables serve the search verbs through `_use_dfa_span`; `match()` does not use them). Subset construction runs at **compile time** over the comptime NFA (byte-equivalence classes bound the per-state work); the transition table (`num_states x 256` ids in the narrowest element type that fits them — `Int8`/`Int16`/`Int32`, `edfa_id_dtype` — packed into a comptime string literal, `static_bytes.mojo`, which lowers to one static `c"..."` global; an `Array` global costs O(n²) to translate) and per-state match/EOL flag bytes materialize as constant data in the binary. The runtime engine is a pure table walk: no lazy construction, no hashing, no `raises` path, no runtime NFA copy in `__init__`. Handles the same three start contexts (pos 0 / after `\n` / mid-line) and EOL flags as the lazy DFA. Three structural passes run at comptime, in order: the DFA is **Hopcroft-minimized** over its byte classes (`_minimize`, initial partition by full flag byte, so states with different EOL flags never merge); states are then permuted so match states occupy ids `[0, num_match_states)` (the per-byte match test is an integer compare, not a flags load) followed by the word-conditional match states; and states that self-loop on all but ≤ 2 bytes (e.g. the `.*` state of `.*x`) are **accelerated** — the walkers SIMD-scan to the next exit byte instead of stepping the table (states carrying `EOL_AT_NEWLINE` or a word-conditional flag are excluded to keep per-byte match tracking), as are **regions** of start states whose rows agree outside a sparse exit set (the look-behind-split restart states of a `\b` pattern). Word boundaries: a state keeps the anchor as a pending member plus the word class of the byte that led to it, resolves it against the next byte's class on the transition, and carries `EDFA_MATCH_IF_WORD` / `_NONWORD` flags the walkers check before consuming (end of input is non-word). Patterns whose determinization exceeds `EDFA_STATE_CAP` (128) states are detected at compile time and stay on the lazy DFA. -The classic table is leftmost-LONGEST (its states are sets), which is exactly what `match()` — Python `fullmatch`, a language-membership question — needs. The search-family verbs (`search`/`finditer`/`findall`/`replace`/`split`) run on a second table instead: the **leftmost-first DFA** (`static_lfdfa.mojo`, `build_lf_dfa`), whose states are priority-ORDERED lists of NFA states (DFS order of the epsilon closure, `out1` before `out2`) with truncation at MATCH and a `restart` bit that folds the unanchored start in as the lowest-priority threads. One unanchored forward walk (`lfdfa_find_end`, the same `edfa_walk_from` walker in the unanchored start states) yields Python's leftmost-first END directly — lazy quantifiers ride this lane too, `<.*?>` stops at its first `>` — and the **reverse DFA** (`static_rdfa.mojo`, `rdfa_find_start`) walks back from that end, never below the previous match end, for the start. The LF table reuses `_edfa_finish` (minimization, match-state permutation, acceleration) and the Sheng masks. A lazy pattern whose LF determinization overflows goes to the backtracker, never the lazy DFA (whose leftmost-longest walk is the wrong engine for `.*?`); a greedy one whose classic table fits but whose LF table overflows runs its search verbs on the backtracker too. The lazy DFA (`_use_lazy_dfa`) backs only patterns whose CLASSIC determinization overflowed, and it re-runs `_lf_end_at` for the leftmost-first end. Before the unanchored scan, `_lf_next_match` tries the first prefilter candidate **anchored** when a cheap anchored engine exists for the shape (`_lf_anchored_classic`: the classic table, when its longest end is the leftmost-first end — one greedy loop at most; `_lf_anchored_sbt`: the backtracker, for lazy patterns whose loops are all simple): a success needs no reverse walk, a failure hands the next candidate to the scan (one attempt per match, so the lane stays linear). When the lane has no filter/Teddy prefix but the pattern carries a **required inner literal** of ≥ 2 bytes (`extract_inner_literal`: `\w+\.txt` must contain ".txt"), `_use_rev_literal` memmems it first (`simd_find_literal_rare`): no occurrence means no match — one SIMD pass instead of the scan — and a comptime-bounded pre-literal gap moves the scan start to `lit_pos - max_offset` (Rust regex's ReverseSuffix/ReverseInner, effects (a)+(b); no leftward walking, so no backscan guard is needed). Tables travel as string literals (see `static_bytes.mojo`), so they are always static data; `EDFA_TABLE_MIN_BYTES` padding is kept for the element-count contract only. +The classic table is leftmost-LONGEST (its states are sets), which is exactly what `match()` — Python `fullmatch`, a language-membership question — needs. The search-family verbs (`search`/`finditer`/`findall`/`replace`/`split`) run on a second table instead: the **leftmost-first DFA** (`static_lfdfa.mojo`, `build_lf_dfa`), whose states are priority-ORDERED lists of NFA states (DFS order of the epsilon closure, `out1` before `out2`) with truncation at MATCH and a `restart` bit that folds the unanchored start in as the lowest-priority threads. One unanchored forward walk (`edfa_match_at` over the LF table, whose start states are the unanchored ones) yields Python's leftmost-first END directly — lazy quantifiers ride this lane too, `<.*?>` stops at its first `>` — and the **reverse DFA** (`static_rdfa.mojo`, `rdfa_find_start`) walks back from that end, never below the previous match end, for the start. The LF table reuses `_edfa_finish` (minimization, match-state permutation, acceleration) and the Sheng masks. A lazy pattern whose LF determinization overflows goes to the backtracker, never the lazy DFA (whose leftmost-longest walk is the wrong engine for `.*?`); a greedy one whose classic table fits but whose LF table overflows runs its search verbs on the backtracker too. The lazy DFA (`_use_lazy_dfa`) backs only patterns whose CLASSIC determinization overflowed, and it re-runs `_lf_end_at` for the leftmost-first end. Before the unanchored scan, `_lf_next_match` tries the first prefilter candidate **anchored** when a cheap anchored engine exists for the shape (`_lf_anchored_classic`: the classic table, when its longest end is the leftmost-first end — one greedy loop at most; `_lf_anchored_sbt`: the backtracker, for lazy patterns whose loops are all simple): a success needs no reverse walk, a failure hands the next candidate to the scan (one attempt per match, so the lane stays linear). When the lane has no filter/Teddy prefix but the pattern carries a **required inner literal** of ≥ 2 bytes (`extract_inner_literal`: `\w+\.txt` must contain ".txt"), `_use_rev_literal` memmems it first (`simd_find_literal_rare`): no occurrence means no match — one SIMD pass instead of the scan — and a comptime-bounded pre-literal gap moves the scan start to `lit_pos - max_offset` (Rust regex's ReverseSuffix/ReverseInner, effects (a)+(b); no leftward walking, so no backscan guard is needed). Tables travel as string literals (see `static_bytes.mojo`), so they are always static data; `EDFA_TABLE_MIN_BYTES` padding is kept for the element-count contract only. **Lazy DFA** (`dfa.mojo`) — fallback DFA engine for patterns that blow the comptime state cap. Builds DFA states on demand from NFA epsilon closures and caches transitions in a 256-entry table per state. Single-pass O(n), no capture overhead. Handles simple line anchors directly: BOL/BOL_MULTILINE resolved in epsilon closure, EOL/EOL_MULTILINE checked at `\n` positions and end-of-input via precomputed flags. At `DFA_STATE_CAP` (4096) runtime states it clears the state cache and continues the walk (the current state is re-interned from its NFA set, the three start states are rebuilt); it only raises `DFA_STATE_CAP` — sending callers to the Pike VM — once it has cleared `MIN_CACHE_CLEARS` (3) times and is still consuming fewer than `MIN_BYTES_PER_STATE` (10) input bytes per state minted since the last clear. diff --git a/emberregex/engine.mojo b/emberregex/engine.mojo index 6c31855..4ecf18e 100644 --- a/emberregex/engine.mojo +++ b/emberregex/engine.mojo @@ -94,11 +94,7 @@ from .static_dfa import ( edfa_match_at, pivot_first_candidate, ) -from .static_lfdfa import ( - build_lf_dfa, - lfdfa_find_end, - sheng_lfdfa_find_end, -) +from .static_lfdfa import build_lf_dfa from .static_rdfa import ( build_reverse_dfa, rdfa_find_start, @@ -1628,15 +1624,15 @@ struct Regex[pattern: String](Copyable, Movable): ](self, input: Span[Byte, origin], pos: Int) -> Int: """Leftmost-first match END at or after `pos`, or -1.""" comptime if Self._use_lf_sheng: - return sheng_lfdfa_find_end[ - lf=Self._lfdfa, + return sheng_match_at[ + d=Self._lfdfa.d, cap=Self._LF_SHENG_CAP, masks=Self._LF_SHENG_MASKS, flags=Self._LFDFA_FLAGS, ](input, pos) else: - return lfdfa_find_end[ - lf=Self._lfdfa, + return edfa_match_at[ + d=Self._lfdfa.d, table=Self._LFDFA_TABLE, flags=Self._LFDFA_FLAGS, ](input, pos) diff --git a/emberregex/sheng.mojo b/emberregex/sheng.mojo index 615a97e..3ace640 100644 --- a/emberregex/sheng.mojo +++ b/emberregex/sheng.mojo @@ -32,8 +32,8 @@ input too short to amortize the shuffle's fixed costs, which walks the same mask table one scalar load per byte instead (`sheng_short_input`, `_sheng_scalar_full_match`). Search-family verbs no longer walk per candidate position: the leftmost-first DFA (static_lfdfa.mojo) runs one -unanchored `sheng_walk_from` pass, so there is no shuffle-engine -search_forward here any more. +unanchored `sheng_match_at` pass over its own table, so there is no +shuffle-engine search_forward here any more. """ from std.collections import Array @@ -365,25 +365,21 @@ def _sheng_walk_impl[ masks: StringLiteral, flags: Array[UInt8, ns], accel: Bool, - s_at0: Int, - s_nl: Int, - s_other: Int, - s_other_w: Int, ](input: Span[Byte, origin], start: Int) -> Int: comptime dead = d.num_states var flg = materialize[flags]() var cur: Int if start == 0: - cur = s_at0 + cur = d.start_at_0 elif input.unsafe_get(start - 1) == CHAR_NEWLINE: - cur = s_nl + cur = d.start_after_nl else: - comptime if s_other_w != s_other: - cur = s_other_w if is_word_byte( + comptime if d.start_other_word != d.start_other: + cur = d.start_other_word if is_word_byte( input.unsafe_get(start - 1) - ) else s_other + ) else d.start_other else: - cur = s_other + cur = d.start_other var cur_vec = _ShuffleIndex(UInt8(cur)) var last_match = -1 @@ -431,7 +427,7 @@ def _sheng_walk_impl[ @always_inline -def sheng_walk_from[ +def sheng_match_at[ origin: Origin, ns: Int, //, @@ -439,59 +435,15 @@ def sheng_walk_from[ cap: Int, masks: StringLiteral, flags: Array[UInt8, ns], - s_at0: Int, - s_nl: Int, - s_other: Int, - s_other_w: Int = s_other, ](input: Span[Byte, origin], start: Int) -> Int: - """Shuffle walk from `start` in explicit start states (mirrors - edfa_walk_from), with the same per-walk accelerated/plain dispatch.""" + """`edfa_match_at` on the shuffle engine, with the same per-walk + accelerated/plain dispatch.""" comptime if _edfa_has_accel(d): comptime W = simd_width_of[DType.uint8]() if len(input) - start >= W: return _sheng_walk_impl[ - d=d, - cap=cap, - masks=masks, - flags=flags, - accel=True, - s_at0=s_at0, - s_nl=s_nl, - s_other=s_other, - s_other_w=s_other_w, + d=d, cap=cap, masks=masks, flags=flags, accel=True ](input, start) return _sheng_walk_impl[ - d=d, - cap=cap, - masks=masks, - flags=flags, - accel=False, - s_at0=s_at0, - s_nl=s_nl, - s_other=s_other, - s_other_w=s_other_w, - ](input, start) - - -@always_inline -def sheng_match_at[ - origin: Origin, - ns: Int, - //, - d: EagerDFA, - cap: Int, - masks: StringLiteral, - flags: Array[UInt8, ns], -](input: Span[Byte, origin], start: Int) -> Int: - """Anchored match at `start` (mirrors edfa_match_at): `sheng_walk_from` - in the DFA's own start states.""" - return sheng_walk_from[ - d=d, - cap=cap, - masks=masks, - flags=flags, - s_at0=d.start_at_0, - s_nl=d.start_after_nl, - s_other=d.start_other, - s_other_w=d.start_other_word, + d=d, cap=cap, masks=masks, flags=flags, accel=False ](input, start) diff --git a/emberregex/static_dfa.mojo b/emberregex/static_dfa.mojo index ddf467d..edf6fd9 100644 --- a/emberregex/static_dfa.mojo +++ b/emberregex/static_dfa.mojo @@ -2394,26 +2394,22 @@ def _edfa_walk_impl[ table: StringLiteral, flags: Array[UInt8, ns], accel: Bool, - s_at0: Int, - s_nl: Int, - s_other: Int, - s_other_w: Int, ](input: Span[Byte, origin], start: Int) -> Int: comptime dt = edfa_id_dtype(d.num_states) var tbl = table.unsafe_ptr().unsafe_bitcast[Scalar[dt]]() var flg = materialize[flags]() var cur: Int if start == 0: - cur = s_at0 + cur = d.start_at_0 elif input.unsafe_get(start - 1) == CHAR_NEWLINE: - cur = s_nl + cur = d.start_after_nl else: - comptime if s_other_w != s_other: - cur = s_other_w if is_word_byte( + comptime if d.start_other_word != d.start_other: + cur = d.start_other_word if is_word_byte( input.unsafe_get(start - 1) - ) else s_other + ) else d.start_other else: - cur = s_other + cur = d.start_other var last_match = -1 if cur < d.num_match_states: @@ -2459,29 +2455,26 @@ def _edfa_walk_impl[ @always_inline -def edfa_walk_from[ +def edfa_match_at[ origin: Origin, ns: Int, //, d: EagerDFA, table: StringLiteral, flags: Array[UInt8, ns], - s_at0: Int, - s_nl: Int, - s_other: Int, - s_other_w: Int = s_other, ](input: Span[Byte, origin], start: Int) -> Int: - """Table walk from `start` in one of the explicit start states - (position 0 / after '\n' / mid-line after a non-word byte / mid-line - after a word byte — the last two coincide unless a word anchor is - live at the start), returning the last position where a match state - (or a resolving EOL / word-boundary flag) was observed, or -1. + """Table walk from `start` in the DFA's own start states (position 0 + / after '\n' / mid-line after a non-word byte / mid-line after a word + byte — the last two coincide unless a word anchor is live at the + start), returning the last position where a match state (or a + resolving EOL / word-boundary flag) was observed, or -1. What that position MEANS depends on the table: over the classic subset construction it is the leftmost-longest end of a match - anchored at `start`; over a leftmost-first table (static_lfdfa.mojo) - it is Python's leftmost-first end, anchored or unanchored according - to which start ids the caller hands over. + anchored at `start` (mirrors LazyDFA.match_at); over a leftmost-first + table (static_lfdfa.mojo), whose start states are the unanchored + ones, it is Python's leftmost-first end of the first match at or + after `start`. Dispatches once per walk between an accelerated and a plain loop: walks that can never reach a full vector chunk take the plain loop @@ -2490,49 +2483,12 @@ def edfa_walk_from[ comptime if _edfa_has_accel(d): comptime W = simd_width_of[DType.uint8]() if len(input) - start >= W: - return _edfa_walk_impl[ - d=d, - table=table, - flags=flags, - accel=True, - s_at0=s_at0, - s_nl=s_nl, - s_other=s_other, - s_other_w=s_other_w, - ](input, start) - return _edfa_walk_impl[ - d=d, - table=table, - flags=flags, - accel=False, - s_at0=s_at0, - s_nl=s_nl, - s_other=s_other, - s_other_w=s_other_w, - ](input, start) - - -@always_inline -def edfa_match_at[ - origin: Origin, - ns: Int, - //, - d: EagerDFA, - table: StringLiteral, - flags: Array[UInt8, ns], -](input: Span[Byte, origin], start: Int) -> Int: - """Anchored match at `start`; returns leftmost-longest end or -1 - (mirrors LazyDFA.match_at). `edfa_walk_from` in the DFA's own start - states.""" - return edfa_walk_from[ - d=d, - table=table, - flags=flags, - s_at0=d.start_at_0, - s_nl=d.start_after_nl, - s_other=d.start_other, - s_other_w=d.start_other_word, - ](input, start) + return _edfa_walk_impl[d=d, table=table, flags=flags, accel=True]( + input, start + ) + return _edfa_walk_impl[d=d, table=table, flags=flags, accel=False]( + input, start + ) @always_inline diff --git a/emberregex/static_lfdfa.mojo b/emberregex/static_lfdfa.mojo index 9f69bd2..a5f4ee3 100644 --- a/emberregex/static_lfdfa.mojo +++ b/emberregex/static_lfdfa.mojo @@ -45,7 +45,6 @@ Continuation closures are memoized per target state in one flat pool. """ from std.bit import count_leading_zeros, count_trailing_zeros -from std.collections import Array from .ast import AnchorKind from .constants import CHAR_NEWLINE @@ -75,9 +74,7 @@ from .static_dfa import ( _word_anchor_bits, WB_PENDING, WB_RESOLVE, - edfa_walk_from, ) -from .sheng import sheng_walk_from # Lanes of a state's ordered-list vector. The last three lanes are the # tail-kind, restart and look-behind markers, so a state holds at most @@ -1090,58 +1087,3 @@ def build_lf_dfa( result.d.start_other_word = pstarts[3] result.valid = True return result^ - - -# --- Runtime walkers --------------------------------------------------------- -# -# Both are the eager table walk (`edfa_walk_from`) in different start -# states — the leftmost-first bookkeeping is entirely in the table. The -# Sheng variants are the shuffle walk over masks built from the same -# table (`sheng_masks_arr(lf.d, ...)`). - - -@always_inline -def lfdfa_find_end[ - origin: Origin, - ns: Int, - //, - lf: LFDFA, - table: StringLiteral, - flags: Array[UInt8, ns], -](input: Span[Byte, origin], start: Int) -> Int: - """Unanchored scan from `start`: the END of the leftmost-first match - beginning at or after `start` (Python `re.search` semantics), or -1. - The start context (position 0 / after '\\n' / mid-line) is read from - `input[start - 1]`.""" - return edfa_walk_from[ - d=lf.d, - table=table, - flags=flags, - s_at0=lf.d.start_at_0, - s_nl=lf.d.start_after_nl, - s_other=lf.d.start_other, - s_other_w=lf.d.start_other_word, - ](input, start) - - -@always_inline -def sheng_lfdfa_find_end[ - origin: Origin, - ns: Int, - //, - lf: LFDFA, - cap: Int, - masks: StringLiteral, - flags: Array[UInt8, ns], -](input: Span[Byte, origin], start: Int) -> Int: - """`lfdfa_find_end` on the shuffle engine.""" - return sheng_walk_from[ - d=lf.d, - cap=cap, - masks=masks, - flags=flags, - s_at0=lf.d.start_at_0, - s_nl=lf.d.start_after_nl, - s_other=lf.d.start_other, - s_other_w=lf.d.start_other_word, - ](input, start) diff --git a/test/test_leftmost_first_dfa.mojo b/test/test_leftmost_first_dfa.mojo index 254bffa..5234bb3 100644 --- a/test/test_leftmost_first_dfa.mojo +++ b/test/test_leftmost_first_dfa.mojo @@ -19,14 +19,11 @@ from emberregex.static_dfa import ( EagerDFA, edfa_flags_arr, edfa_id_dtype, + edfa_match_at, edfa_table_str, ) from emberregex.static_rdfa import rdfa_find_start -from emberregex.static_lfdfa import ( - LF_LIST_CAP, - build_lf_dfa, - lfdfa_find_end, -) +from emberregex.static_lfdfa import LF_LIST_CAP, build_lf_dfa from std.benchmark import keep from std.testing import assert_true, assert_false, assert_equal, TestSuite from std.time import perf_counter_ns @@ -694,11 +691,11 @@ def test_wide_list_signature_renumbering() raises: comptime flags = edfa_flags_arr[lf.d.num_states](lf.d) var input = String("!!!!!!!!!!!!!!!!!!!!q!!") var bytes = input.as_bytes() - assert_equal(lfdfa_find_end[lf=lf, table=table, flags=flags](bytes, 0), 21) - assert_equal(lfdfa_find_end[lf=lf, table=table, flags=flags](bytes, 21), -1) + assert_equal(edfa_match_at[d=lf.d, table=table, flags=flags](bytes, 0), 21) + assert_equal(edfa_match_at[d=lf.d, table=table, flags=flags](bytes, 21), -1) var input2 = String("!!!!!!!!!!!!!!!!!!!!7!!") var bytes2 = input2.as_bytes() - assert_equal(lfdfa_find_end[lf=lf, table=table, flags=flags](bytes2, 0), 21) + assert_equal(edfa_match_at[d=lf.d, table=table, flags=flags](bytes2, 0), 21) # And on the engine: a class arm keeps Teddy off, so the same lists # drive search/findall through the lane. comptime W = Regex[_WIDE_ALT + "|[!?]{2}"] diff --git a/test/test_word_boundary_dfa.mojo b/test/test_word_boundary_dfa.mojo index bebc8ec..061ed42 100644 --- a/test/test_word_boundary_dfa.mojo +++ b/test/test_word_boundary_dfa.mojo @@ -27,7 +27,7 @@ from emberregex.static_dfa import ( edfa_match_at, edfa_table_str, ) -from emberregex.static_lfdfa import LFDFA, build_lf_dfa, lfdfa_find_end +from emberregex.static_lfdfa import LFDFA, build_lf_dfa from emberregex.static_rdfa import ( build_reverse_dfa, rdfa_find_start, @@ -274,7 +274,9 @@ def _forced_lane_check[p: StaticString](input: String, label: String) raises: var pos = 0 var i = 0 while pos <= n: - var end = lfdfa_find_end[lf=lf, table=ltbl, flags=lfl](bytes, pos) + var end = edfa_match_at[d=lf.d, table=ltbl, flags=lfl]( + bytes, pos + ) if end < 0: break var start = rdfa_find_start[d=rd, table=rtbl, flags=rfl]( From 52fd7743dc0c71aabb5a19ce4df147b827a2e4c7 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:05:41 -0600 Subject: [PATCH 19/65] cleanup: drop the unused InnerLiteral.is_suffix and its bookkeeping Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/optimize.mojo | 18 +++----------- test/test_reverse_literal.mojo | 45 ++++++++++++++++------------------ 2 files changed, 25 insertions(+), 38 deletions(-) diff --git a/emberregex/optimize.mojo b/emberregex/optimize.mojo index 2c3ae6a..8baf7f1 100644 --- a/emberregex/optimize.mojo +++ b/emberregex/optimize.mojo @@ -631,7 +631,7 @@ def extract_literal_suffix(nfa: NFA) -> List[UInt8]: # keeps the consuming lanes far below this. comptime INNER_LIT_MAX_STATES = 512 # Longest literal kept. Any prefix of a required run is itself required, -# so truncation is sound — the truncated run merely stops being a suffix. +# so truncation is sound. comptime INNER_LIT_MAX_LEN = 16 # Alternation nesting the walk resolves before giving up. comptime _INNER_MAX_DEPTH = 12 @@ -647,11 +647,8 @@ struct InnerLiteral(Copyable, Movable): fixed offset 0 are excluded — those belong to the prefix scanners (extract_literal_prefix / extract_filter_prefix / extract_alt_prefix). - `is_suffix` marks a run that ends every match (no bytes are consumed - after it); it is extracted and pinned but unused by the engine until - a suffix end-window verifier (effect (c)) exists. `valid` requires a - run of >= 2 bytes: a single required byte is already covered by - extract_required_byte. + `valid` requires a run of >= 2 bytes: a single required byte is + already covered by extract_required_byte. The engine uses this as a prefilter (Rust regex's ReverseSuffix / ReverseInner, effects (a)+(b)): no occurrence of `bytes` at or after @@ -664,7 +661,6 @@ struct InnerLiteral(Copyable, Movable): var caseless: List[Bool] var min_offset: Int var max_offset: Int - var is_suffix: Bool def __init__(out self): self.valid = False @@ -672,7 +668,6 @@ struct InnerLiteral(Copyable, Movable): self.caseless = List[Bool]() self.min_offset = 0 self.max_offset = 0 - self.is_suffix = False @always_inline @@ -890,7 +885,6 @@ def extract_inner_literal(nfa: NFA, cyclic: List[Bool]) -> InnerLiteral: var buf_c = List[Bool]() var buf_min = 0 var buf_max = 0 - var suffix_flag = False # the LAST closed run abutted MATCH var visited = SIMD[DType.uint8, _INNER_BITS](0) var s = nfa.start @@ -944,7 +938,6 @@ def extract_inner_literal(nfa: NFA, cyclic: List[Bool]) -> InnerLiteral: run_cl.append(buf_c^) buf_b = List[UInt8]() buf_c = List[Bool]() - suffix_flag = kind == NFAStateKind.MATCH if kind == NFAStateKind.MATCH: break @@ -1000,13 +993,12 @@ def extract_inner_literal(nfa: NFA, cyclic: List[Bool]) -> InnerLiteral: break # A bailed walk can leave a run open; its bytes were established from - # mandatory states, so keep it (suffix unknown -> False). + # mandatory states, so keep it. if len(buf_b) > 0: run_min.append(buf_min) run_max.append(buf_max) run_bytes.append(buf_b^) run_cl.append(buf_c^) - suffix_flag = False # Selection: drop fixed-offset-0 runs, require length >= 2, prefer # the rarest (then the longer) run. @@ -1038,14 +1030,12 @@ def extract_inner_literal(nfa: NFA, cyclic: List[Bool]) -> InnerLiteral: if best < 0: return res^ - var truncated = len(run_bytes[best]) > INNER_LIT_MAX_LEN var m = min(len(run_bytes[best]), INNER_LIT_MAX_LEN) for k in range(m): res.bytes.append(run_bytes[best][k]) res.caseless.append(run_cl[best][k]) res.min_offset = run_min[best] res.max_offset = run_max[best] - res.is_suffix = best == len(run_bytes) - 1 and suffix_flag and not truncated res.valid = True return res^ diff --git a/test/test_reverse_literal.mojo b/test/test_reverse_literal.mojo index ae857c7..55ba4db 100644 --- a/test/test_reverse_literal.mojo +++ b/test/test_reverse_literal.mojo @@ -34,7 +34,6 @@ def _assert_inner[ expect: StaticString, min_off: Int, max_off: Int, - suffix: Bool, ]() raises: comptime il = Regex[p]._inner_lit assert_true(il.valid, String("inner literal missing for ", p)) @@ -47,7 +46,6 @@ def _assert_inner[ assert_equal(Int(bi), Int(ei), String("byte ", i, " of ", p)) assert_equal(il.min_offset, min_off, String("min_offset for ", p)) assert_equal(il.max_offset, max_off, String("max_offset for ", p)) - assert_equal(il.is_suffix, suffix, String("is_suffix for ", p)) def _assert_no_inner[p: StaticString]() raises: @@ -58,21 +56,21 @@ def _assert_no_inner[p: StaticString]() raises: def test_extraction_suffix_after_loop() raises: # The canonical reverse-suffix shape: an unbounded loop, then a # required literal ending the match. - _assert_inner["\\w+\\.txt", ".txt", 1, -1, True]() - _assert_inner["\\d+\\.txt", ".txt", 1, -1, True]() + _assert_inner["\\w+\\.txt", ".txt", 1, -1]() + _assert_inner["\\d+\\.txt", ".txt", 1, -1]() # ANY-based loop. - _assert_inner[".+\\.txt", ".txt", 1, -1, True]() + _assert_inner[".+\\.txt", ".txt", 1, -1]() # Literal run preceded by a loop plus an exact byte; the run absorbs # the byte ("z.txt"), so the gap is the loop alone. - _assert_inner["[ab]+z\\.txt", "z.txt", 1, -1, True]() + _assert_inner["[ab]+z\\.txt", "z.txt", 1, -1]() def test_extraction_inner_run() raises: # The reverse-inner shape: the run continues into more consuming - # states, so it is not a suffix. - _assert_inner["[a-z]+://[^ ]+", "://", 1, -1, False]() - # Same run, nothing after it: a suffix. - _assert_inner["[a-z]+://", "://", 1, -1, True]() + # states. + _assert_inner["[a-z]+://[^ ]+", "://", 1, -1]() + # Same run, nothing after it (the reverse-suffix shape). + _assert_inner["[a-z]+://", "://", 1, -1]() def test_extraction_alternation_gap() raises: @@ -80,13 +78,13 @@ def test_extraction_alternation_gap() raises: # alternations are Teddy-owned (see test_strategy_on_for_bounded_gap # for the shapes that actually run effect (b)). # Both arms consume exactly 3 bytes: the gap is bounded. - _assert_inner["(foo|bar)\\.txt", ".txt", 3, 3, True]() - _assert_inner["(?:foo|bar)\\.txt", ".txt", 3, 3, True]() - # Arms of different lengths: min 1, max 2; trailing charset means the - # run is not a suffix. - _assert_inner["(a|bb)cde[0-9]", "cde", 1, 2, False]() + _assert_inner["(foo|bar)\\.txt", ".txt", 3, 3]() + _assert_inner["(?:foo|bar)\\.txt", ".txt", 3, 3]() + # Arms of different lengths: min 1, max 2; the trailing charset + # closes the run. + _assert_inner["(a|bb)cde[0-9]", "cde", 1, 2]() # Multi-way alternation (a chained SPLIT tree). - _assert_inner["(?:a|bb|ccc)\\.txt", ".txt", 1, 3, True]() + _assert_inner["(?:a|bb|ccc)\\.txt", ".txt", 1, 3]() # The only mandatory run after the alternation is one byte long: # a single required byte is `required_byte`'s territory, not a # literal worth a memmem. @@ -98,10 +96,10 @@ def test_extraction_bounded_counted_gap() raises: # compose min/max through the alternation walk. These four shapes # also HOLD the strategy at the engine level (unlike the literal # alternations above) — they are the effect-(b) test fleet. - _assert_inner["[ab]{0,3}foo", "foo", 0, 3, True]() - _assert_inner["[0-9]{2,5}xy", "xy", 2, 5, True]() - _assert_inner[".{0,2}foo", "foo", 0, 2, True]() - _assert_inner["[ab]?[cd]?foo", "foo", 0, 2, True]() + _assert_inner["[ab]{0,3}foo", "foo", 0, 3]() + _assert_inner["[0-9]{2,5}xy", "xy", 2, 5]() + _assert_inner[".{0,2}foo", "foo", 0, 2]() + _assert_inner["[ab]?[cd]?foo", "foo", 0, 2]() def test_extraction_rejects_prefix_and_short_runs() raises: @@ -121,7 +119,7 @@ def test_extraction_rejects_prefix_and_short_runs() raises: def test_extraction_prefix_run_skipped_inner_kept() raises: # "ab" is the pattern's literal prefix (fixed offset 0) — skipped; # "cd" after the loop is the inner literal. - _assert_inner["ab\\d+cd", "cd", 3, -1, True]() + _assert_inner["ab\\d+cd", "cd", 3, -1]() def test_extraction_caseless() raises: @@ -138,14 +136,13 @@ def test_extraction_caseless() raises: assert_true(ci) assert_equal(il.min_offset, 1) assert_equal(il.max_offset, -1) - assert_true(il.is_suffix) def test_extraction_prefers_rarest_run() raises: # Two mandatory runs; "qux" ('q' is rarer than anything in "the") # wins regardless of order. - _assert_inner["[0-9]+the[0-9]+qux[0-9]+", "qux", 5, -1, False]() - _assert_inner["[0-9]+qux[0-9]+the[0-9]+", "qux", 1, -1, False]() + _assert_inner["[0-9]+the[0-9]+qux[0-9]+", "qux", 5, -1]() + _assert_inner["[0-9]+qux[0-9]+the[0-9]+", "qux", 1, -1]() # --- Strategy selection ------------------------------------------------------ From 22fe9ff267c5b4577d37e17dd05a02658ea5e8c4 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:06:27 -0600 Subject: [PATCH 20/65] cleanup: one shared compile_once for the compile-time tools comptime_regex_stages, comptime_pattern_probe, comptime_stages and compile_dashboard each wrote a source file, timed `mojo build` and checked the return code. tools/timed_build.py now does that once; each tool keeps its own flags, cache policy, timeout and error reporting. The pattern probe stays separate: it builds against the precompiled package and ranks many patterns, where the stages ladder builds against the source tree. Co-Authored-By: Claude Opus 5.5 (1M context) --- tools/compile_dashboard.py | 25 ++++++------------- tools/comptime_pattern_probe.py | 33 ++++++------------------- tools/comptime_regex_stages.py | 28 ++++----------------- tools/comptime_stages.py | 43 +++++++++++---------------------- tools/timed_build.py | 33 +++++++++++++++++++++++++ 5 files changed, 67 insertions(+), 95 deletions(-) create mode 100644 tools/timed_build.py diff --git a/tools/compile_dashboard.py b/tools/compile_dashboard.py index ae5cb00..909c85a 100644 --- a/tools/compile_dashboard.py +++ b/tools/compile_dashboard.py @@ -12,10 +12,10 @@ """ import os -import subprocess import sys import tempfile -import time + +from timed_build import compile_once # Default ladder stops at 32: the N=64 mixed rung measured past 27 # minutes (2026-07-23, table in MULTIPATTERN_PLAN.md) and the decision @@ -55,25 +55,16 @@ def main() raises: def measure(n: int, workdir: str) -> tuple[float, int] | None: - src = os.path.join(workdir, f"set_{n}.mojo") - out = os.path.join(workdir, f"set_{n}") - with open(src, "w") as f: - f.write(gen_source(n)) - t0 = time.monotonic() - try: - ret = subprocess.run( - ["pixi", "run", "mojo", "build", "-I", ".", src, "-o", out], - capture_output=True, - text=True, - timeout=TIMEOUT_S, - ) - except subprocess.TimeoutExpired: + elapsed, ret = compile_once( + gen_source(n), workdir, f"set_{n}", flags=("-I", "."), + timeout=TIMEOUT_S, mojo=("pixi", "run", "mojo"), + ) + if elapsed is None: return None - elapsed = time.monotonic() - t0 if ret.returncode: print(ret.stderr, file=sys.stderr) raise SystemExit(f"compile failed for N={n}") - return elapsed, os.path.getsize(out) + return elapsed, os.path.getsize(os.path.join(workdir, f"set_{n}")) if __name__ == "__main__": diff --git a/tools/comptime_pattern_probe.py b/tools/comptime_pattern_probe.py index d1711d4..77a3d09 100644 --- a/tools/comptime_pattern_probe.py +++ b/tools/comptime_pattern_probe.py @@ -21,13 +21,11 @@ import json import os import re -import shutil -import subprocess import sys import tempfile -import time -ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +from timed_build import ROOT, compile_once + LIT = re.compile(r'Regex\["((?:[^"\\]|\\.)*)"\]') TEMPLATE = """from emberregex import Regex @@ -37,25 +35,6 @@ def main() raises: """ -def compile_once(body, include, cache, work): - src = os.path.join(work, "probe.mojo") - out = os.path.join(work, "probe.bin") - with open(src, "w") as f: - f.write(body) - shutil.rmtree(cache, ignore_errors=True) - os.makedirs(cache) - env = dict(os.environ, MODULAR_CACHE_DIR=cache) - t0 = time.time() - r = subprocess.run( - ["mojo", "build", "-D", "ASSERT=all", "-I", include, src, "-o", out], - cwd=ROOT, - env=env, - capture_output=True, - text=True, - ) - return time.time() - t0, r.returncode, r.stderr[-400:] - - def main(): ap = argparse.ArgumentParser() ap.add_argument("files", nargs="+") @@ -74,13 +53,15 @@ def main(): pats.append(m.group(1)) rows = [] for p in pats: - dt, rc, err = compile_once( - TEMPLATE.replace("{P}", p), args.include, cache, work + dt, r = compile_once( + TEMPLATE.replace("{P}", p), work, + flags=("-D", "ASSERT=all", "-I", args.include), cache=cache, ) + rc = r.returncode rows.append({"pattern": p, "seconds": round(dt, 1), "rc": rc}) print(f"{dt:7.1f}s rc={rc} {p}", flush=True) if rc: - sys.stderr.write(err + "\n") + sys.stderr.write(r.stderr[-400:] + "\n") rows.sort(key=lambda r: -r["seconds"]) all_rows[f] = rows print(f"== {f}: {len(rows)} patterns, sum {sum(r['seconds'] for r in rows):.1f}s ==") diff --git a/tools/comptime_regex_stages.py b/tools/comptime_regex_stages.py index 2b76923..960c54d 100644 --- a/tools/comptime_regex_stages.py +++ b/tools/comptime_regex_stages.py @@ -28,13 +28,10 @@ import argparse import os -import shutil -import subprocess import sys import tempfile -import time -ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +from timed_build import compile_once # Each stage is a program body; {P} is replaced with the pattern literal. # Ordered so adjacent diffs isolate one stage of the pipeline. @@ -109,26 +106,11 @@ def mojo_literal(pattern: str) -> str: return '"' + pattern.replace("\\", "\\\\").replace('"', '\\"') + '"' -def compile_once(body: str, cache_dir: str, work: str) -> float: - src = os.path.join(work, "probe.mojo") - out = os.path.join(work, "probe.bin") - with open(src, "w") as f: - f.write(body) - shutil.rmtree(cache_dir, ignore_errors=True) - os.makedirs(cache_dir) - env = dict(os.environ, MODULAR_CACHE_DIR=cache_dir) - t0 = time.time() - r = subprocess.run( - ["mojo", "build", "-D", "ASSERT=all", "-I", ".", src, "-o", out], - cwd=ROOT, - env=env, - capture_output=True, - text=True, - ) - dt = time.time() - t0 +def compile_stage(body: str, cache_dir: str, work: str) -> float: + dt, r = compile_once(body, work, flags=("-D", "ASSERT=all", "-I", "."), cache=cache_dir) if r.returncode != 0: sys.stderr.write(r.stderr[-1500:] + "\n") - raise SystemExit(f"stage failed to compile (see stderr above)") + raise SystemExit("stage failed to compile (see stderr above)") return dt @@ -145,7 +127,7 @@ def main() -> None: for name, tmpl in STAGES: body = tmpl.replace("{P}", lit) best = min( - compile_once(body, cache, work) for _ in range(args.repeat) + compile_stage(body, cache, work) for _ in range(args.repeat) ) times[name] = best print(f" {name:8s} {best:7.1f}s", flush=True) diff --git a/tools/comptime_stages.py b/tools/comptime_stages.py index 355a393..10c05db 100644 --- a/tools/comptime_stages.py +++ b/tools/comptime_stages.py @@ -32,12 +32,11 @@ """ import argparse -import os import statistics -import subprocess import sys import tempfile -import time + +from timed_build import compile_once TIMEOUT_S = 1800 @@ -156,28 +155,6 @@ def source_for(pats, decls, body): ) -def compile_once(src_text, workdir, tag): - src = os.path.join(workdir, f"{tag}.mojo") - with open(src, "w") as f: - f.write(src_text) - t0 = time.monotonic() - try: - ret = subprocess.run( - ["pixi", "run", "mojo", "build", "-I", ".", src, - "-o", os.path.join(workdir, tag)], - capture_output=True, text=True, timeout=TIMEOUT_S, - ) - except subprocess.TimeoutExpired: - return None, "TIMEOUT" - dt = time.monotonic() - t0 - if ret.returncode != 0: - # Surface the first real error — a stage that stops compiling is a - # broken harness, not a fast stage. - err = [l for l in ret.stderr.splitlines() if "error:" in l] - return None, (err[0][:160] if err else "FAILED") - return dt, None - - def main(): ap = argparse.ArgumentParser() ap.add_argument("sets", nargs="*", default=list(SETS)) @@ -198,11 +175,19 @@ def main(): times = [] err = None for _ in range(args.repeat): - dt, e = compile_once( - source_for(pats, decls, body), workdir, tag + dt, ret = compile_once( + source_for(pats, decls, body), workdir, tag, + flags=("-I", "."), timeout=TIMEOUT_S, + mojo=("pixi", "run", "mojo"), ) - if e: - err = e + if dt is None: + err = "TIMEOUT" + break + if ret.returncode != 0: + # Surface the first real error — a stage that stops + # compiling is a broken harness, not a fast stage. + e = [l for l in ret.stderr.splitlines() if "error:" in l] + err = e[0][:160] if e else "FAILED" break times.append(dt) if err: diff --git a/tools/timed_build.py b/tools/timed_build.py new file mode 100644 index 0000000..a9ad8cc --- /dev/null +++ b/tools/timed_build.py @@ -0,0 +1,33 @@ +"""The one measurement the compile-time tools share: time a `mojo build`.""" + +import os +import shutil +import subprocess +import time + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def compile_once(body, work, name="probe", flags=(), cache=None, timeout=None, + mojo=("mojo",)): + """Write `body` to `work/.mojo` and time `mojo build` of it (from + ROOT, output `work/`). With `cache`, the build gets that private + MODULAR_CACHE_DIR, cleared first: a solo-cold compile. Returns + (seconds, CompletedProcess), or (None, None) past `timeout`.""" + src = os.path.join(work, name + ".mojo") + with open(src, "w") as f: + f.write(body) + env = None + if cache: + shutil.rmtree(cache, ignore_errors=True) + os.makedirs(cache) + env = dict(os.environ, MODULAR_CACHE_DIR=cache) + t0 = time.monotonic() + try: + r = subprocess.run( + [*mojo, "build", *flags, src, "-o", os.path.join(work, name)], + cwd=ROOT, env=env, capture_output=True, text=True, timeout=timeout, + ) + except subprocess.TimeoutExpired: + return None, None + return time.monotonic() - t0, r From 2430806e4d8c611b82826c8ad6c08f2d17fe201e Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:06:56 -0600 Subject: [PATCH 21/65] cleanup: build span-only results through Regex._span_result Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/engine.mojo | 75 ++++++++++-------------------------------- 1 file changed, 18 insertions(+), 57 deletions(-) diff --git a/emberregex/engine.mojo b/emberregex/engine.mojo index 733d81c..97bce1b 100644 --- a/emberregex/engine.mojo +++ b/emberregex/engine.mojo @@ -2220,12 +2220,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( comptime sb = Self._sandwich.suffix[i] if ptr[unsafe_offset=input_len - suffix_len + i] != sb: return MatchResult[Self._num_slots].no_match() - return MatchResult[Self._num_slots]( - matched=True, - start=0, - end=input_len, - slots=Array[Int, Self._num_slots](fill=-1), - ) + return Self._span_result(0, input_len) elif Self._strategy.use_simd_literal: var lit = rebind[TypeForPrefixLength[Self._strategy.prefix_len]]( self._simd_lit @@ -2235,22 +2230,12 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( width=Self._strategy.prefix_len ]() if chunk == lit: - return MatchResult[Self._num_slots]( - matched=True, - start=0, - end=Self._strategy.prefix_len, - slots=Array[Int, Self._num_slots](fill=-1), - ) + return Self._span_result(0, Self._strategy.prefix_len) return MatchResult[Self._num_slots].no_match() elif Self._strategy.use_dfa: try: if self._dfa_full_match(input): - return MatchResult[Self._num_slots]( - matched=True, - start=0, - end=input.byte_length(), - slots=Array[Int, Self._num_slots](fill=-1), - ) + return Self._span_result(0, input.byte_length()) return MatchResult[Self._num_slots].no_match() except: # Only the lazy DFA can raise here (DFA_STATE_CAP): for @@ -2335,12 +2320,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( var pos = simd_find_literal(input_bytes, lit, 0) if pos < 0: return MatchResult[Self._num_slots].no_match() - return MatchResult[Self._num_slots]( - matched=True, - start=pos, - end=pos + Self._strategy.prefix_len, - slots=Array[Int, Self._num_slots](fill=-1), - ) + return Self._span_result(pos, pos + Self._strategy.prefix_len) elif Self._use_lf_lane: # The same two-line prologue opens every leftmost-first lane # verb: the walk's state lives in the verb's frame and the @@ -2367,11 +2347,8 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( comptime if Self._strategy.start_anchor == AnchorKind.BOL: var match_end = self._dfa_match_at(input_bytes, 0) if match_end >= 0: - return MatchResult[Self._num_slots]( - matched=True, - start=0, - end=self._lf_end_at(input_bytes, 0, match_end), - slots=Array[Int, Self._num_slots](fill=-1), + return Self._span_result( + 0, self._lf_end_at(input_bytes, 0, match_end) ) return MatchResult[Self._num_slots].no_match() @@ -2382,13 +2359,9 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( while pos <= input_len: var match_end = self._dfa_match_at(input_bytes, pos) if match_end >= 0: - return MatchResult[Self._num_slots]( - matched=True, - start=pos, - end=self._lf_end_at( - input_bytes, pos, match_end - ), - slots=Array[Int, Self._num_slots](fill=-1), + return Self._span_result( + pos, + self._lf_end_at(input_bytes, pos, match_end), ) var nl = simd_find_byte(input_bytes, CHAR_NEWLINE, pos) if nl < 0: @@ -2407,13 +2380,11 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( return MatchResult[Self._num_slots].no_match() var match_end = self._dfa_match_at(input_bytes, pos) if match_end >= 0: - return MatchResult[Self._num_slots]( - matched=True, - start=pos, - end=self._lf_end_at( + return Self._span_result( + pos, + self._lf_end_at( input_bytes, pos, match_end ), - slots=Array[Int, Self._num_slots](fill=-1), ) pos = _scan_bump[Self._is_unicode](input_bytes, pos) else: @@ -2421,13 +2392,11 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( input_bytes, pos ) if range[0] >= 0: - return MatchResult[Self._num_slots]( - matched=True, - start=range[0], - end=self._lf_end_at( + return Self._span_result( + range[0], + self._lf_end_at( input_bytes, range[0], range[1] ), - slots=Array[Int, Self._num_slots](fill=-1), ) return MatchResult[Self._num_slots].no_match() return MatchResult[Self._num_slots].no_match() @@ -3196,11 +3165,8 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( if literal_replacement: output += replacement else: - var match_result = MatchResult[Self._num_slots]( - matched=True, - start=pos, - end=pos + Self._strategy.prefix_len, - slots=Array[Int, Self._num_slots](fill=-1), + var match_result = Self._span_result( + pos, pos + Self._strategy.prefix_len ) output += self._expand_replacement( input_bytes, match_result, replacement @@ -3329,12 +3295,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( if literal_replacement: output += replacement else: - var match_result = MatchResult[Self._num_slots]( - matched=True, - start=start, - end=end, - slots=Array[Int, Self._num_slots](fill=-1), - ) + var match_result = Self._span_result(start, end) output += self._expand_replacement( input_bytes, match_result, replacement ) From 6914f28c351dbb0eb39668eabe3a16c4ffeb86f9 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:07:02 -0600 Subject: [PATCH 22/65] cleanup: collapse Pike VM step arms and MATCH scans, drop unused max_pos One _add_state call for CHAR/ANY/CHARSET, one MATCH scan for the pinned and unpinned modes (the cut only when unpinned); max_pos was never passed by any caller. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/executor.mojo | 111 ++++++++++++--------------------------- 1 file changed, 35 insertions(+), 76 deletions(-) diff --git a/emberregex/executor.mojo b/emberregex/executor.mojo index 451601c..3410ab3 100644 --- a/emberregex/executor.mojo +++ b/emberregex/executor.mojo @@ -117,14 +117,12 @@ struct PikeVM[num_slots: Int](Copyable): input: Span[Byte, origin], start_pos: Int, mut bufs: _VMBuffers, - max_pos: Int = -1, full: Bool = False, unanchored: Bool = False, end_at: Int = -1, ) -> MatchResult[Self.num_slots]: """Core NFA simulation using pre-allocated buffers. - If max_pos >= 0, limits processing to positions < max_pos. If full is True, MATCH only accepts at end of input (fullmatch); otherwise the VM implements leftmost-first (Python re) semantics. If unanchored is True, a fresh start-state thread is injected at @@ -134,16 +132,14 @@ struct PikeVM[num_slots: Int](Copyable): If end_at >= 0, MATCH accepts only at exactly that position (the first thread in priority order there wins, as in fullmatch) and the simulation stops there — but anchors and word boundaries - still see the REAL input: `max_pos` would truncate `input_len`, - so `$` would hold at the pin and `\b` would see no byte after - it, which is wrong for the DFA-span capture lane (engine.mojo - `_span_fill_slots`), whose span ends mid-input. + still see the REAL input: truncating it at the pin would make `$` + hold there and leave `\b` no byte after it, which is wrong for + the DFA-span capture lane (engine.mojo `_span_fill_slots`), whose + span ends mid-input. """ var input_len = len(input) - if max_pos >= 0 and max_pos < input_len: - input_len = max_pos # Where the simulation stops and fullmatch-style acceptance - # applies: the end pin, else the (possibly truncated) input end. + # applies: the end pin, else the input end. var stop = input_len var pinned = full if end_at >= 0 and end_at <= input_len: @@ -178,34 +174,15 @@ struct PikeVM[num_slots: Int](Copyable): var pos = start_pos while True: - # Check for match states - if pinned: - # Fullmatch / end pin: MATCH only accepts at `stop`; the - # first (highest-priority) thread that reached it wins. - if pos >= stop: - for i in range(len(bufs.current_states)): - if ( - self.nfa.states.unsafe_get( - bufs.current_states.unsafe_get(i) - ).kind - == NFAStateKind.MATCH - ): - matched = True - best_match_end = pos - for s in range(Self._stride): - bufs.best_slots.unsafe_set( - s, - bufs.current_slot_data.unsafe_get( - i * Self._stride + s - ), - ) - break - else: - # Leftmost-first (Python re semantics): the first thread in - # priority order to reach MATCH beats every lower-priority - # thread, so record it and cut those threads. Surviving - # higher-priority threads may still override with a match - # they reach later (e.g. the greedy arm of `a*`). + # Check for match states. Fullmatch / end pin: MATCH only + # accepts at `stop`, where the first (highest-priority) thread + # that reached it wins. Otherwise leftmost-first (Python re + # semantics): the first thread in priority order to reach + # MATCH beats every lower-priority thread, so record it and cut + # those threads. Surviving higher-priority threads may still + # override with a match they reach later (e.g. the greedy arm + # of `a*`). + if not pinned or pos >= stop: for i in range(len(bufs.current_states)): if ( self.nfa.states.unsafe_get( @@ -222,8 +199,9 @@ struct PikeVM[num_slots: Int](Copyable): i * Self._stride + s ), ) - bufs.current_states.resize(i, 0) - bufs.current_slot_data.resize(i * Self._stride, 0) + if not pinned: + bufs.current_states.resize(i, 0) + bufs.current_slot_data.resize(i * Self._stride, 0) break if pos >= stop: @@ -247,46 +225,27 @@ struct PikeVM[num_slots: Int](Copyable): s, bufs.current_slot_data.unsafe_get(base + s) ) + var ok = False if kind == NFAStateKind.CHAR: - if ch == state.char_value: - self._add_state( - bufs.next_states, - bufs.next_slot_data, - bufs.gen, - next_gen, - out1, - bufs.temp_slots, - input, - input_len, - pos + 1, - ) + ok = ch == state.char_value elif kind == NFAStateKind.ANY: - if ch != UInt32(CHAR_NEWLINE): - self._add_state( - bufs.next_states, - bufs.next_slot_data, - bufs.gen, - next_gen, - out1, - bufs.temp_slots, - input, - input_len, - pos + 1, - ) + ok = ch != UInt32(CHAR_NEWLINE) elif kind == NFAStateKind.CHARSET: - var cs_idx = state.charset_index - if self.nfa.charsets.unsafe_get(cs_idx).contains(ch): - self._add_state( - bufs.next_states, - bufs.next_slot_data, - bufs.gen, - next_gen, - out1, - bufs.temp_slots, - input, - input_len, - pos + 1, - ) + ok = self.nfa.charsets.unsafe_get( + state.charset_index + ).contains(ch) + if ok: + self._add_state( + bufs.next_states, + bufs.next_slot_data, + bufs.gen, + next_gen, + out1, + bufs.temp_slots, + input, + input_len, + pos + 1, + ) # Unanchored: seed a fresh lowest-priority thread at the next # position while no match is recorded (earlier-start threads From 716ad27be7f542332eb442beb4d164bc3e5d5bfb Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:07:18 -0600 Subject: [PATCH 23/65] cleanup: one eager/Sheng walker pair with a comptime cap tier _sheng_walk_impl and _sheng_full_match_impl were _edfa_walk_impl and _edfa_full_match_impl with a shuffle step (and a state-vector resync after a region skip); cap == 0 keeps the table step. _sheng_step moves to simd_kernels next to the tbl lookups it wraps. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/sheng.mojo | 206 +++-------------------------------- emberregex/simd_kernels.mojo | 25 +++++ emberregex/static_dfa.mojo | 125 +++++++++++++++------ 3 files changed, 132 insertions(+), 224 deletions(-) diff --git a/emberregex/sheng.mojo b/emberregex/sheng.mojo index 3ace640..1419c90 100644 --- a/emberregex/sheng.mojo +++ b/emberregex/sheng.mojo @@ -25,9 +25,10 @@ x86 pshufb has no multi-register form, so SHENG_STATE_CAP falls back to 16 there (HAS_WIDE_BYTE_SHUFFLE); selection additionally requires HAS_FAST_BYTE_SHUFFLE, and other targets keep the eager table walk. -The walkers mirror the eager DFA walkers exactly (start contexts, EOL -flags, acceleration, per-walk accel dispatch) — only the transition -mechanism differs. The one exception is an anchored full match on an +The walkers ARE the eager DFA walkers (static_dfa.mojo's +`_edfa_walk_impl` / `_edfa_full_match_impl` at a `cap` tier: start +contexts, EOL flags, acceleration, per-walk accel dispatch) — only the +transition mechanism differs. The one exception is an anchored full match on an input too short to amortize the shuffle's fixed costs, which walks the same mask table one scalar load per byte instead (`sheng_short_input`, `_sheng_scalar_full_match`). Search-family verbs no longer walk per candidate @@ -39,26 +40,15 @@ shuffle-engine search_forward here any more. from std.collections import Array from std.sys import simd_width_of -from .constants import CHAR_NEWLINE, is_word_byte from .static_bytes import filled_string from .static_dfa import ( EDFA_EOL_AT_END, - EDFA_EOL_AT_NEWLINE, - EDFA_MATCH_IF_WORD, EagerDFA, - _edfa_accel_skip, + _edfa_full_match_impl, _edfa_has_accel, - _edfa_has_region, - _edfa_region_skip, -) -from .simd_kernels import ( - HAS_WIDE_BYTE_SHUFFLE, - NIBBLE_TABLE_SIZE, - _ShuffleIndex, - nibble_lookup, - table_lookup_32, - table_lookup_64, + edfa_match_at, ) +from .simd_kernels import HAS_WIDE_BYTE_SHUFFLE, NIBBLE_TABLE_SIZE # Widest tbl tier this target can do in one instruction (see module # docstring) — an algorithmic constant, NOT a platform vector width. @@ -141,36 +131,6 @@ def sheng_masks_str[cap: Int](d: EagerDFA, enabled: Bool) -> String: return out^ -@always_inline -def _sheng_step[ - cap: Int -](masks: StringLiteral, b: Byte, state_vec: _ShuffleIndex) -> _ShuffleIndex: - """One transition: shuffle the byte's mask by the state vector (the - state id broadcast across the index register; only lane 0 is ever - read back, and its width is independent of the mask width). - - Each branch loads and shuffles at the literal tier width `cap`: only the tier this DFA needs is emitted, - and the NEON-only tiers are never elaborated where cap is always - NIBBLE_TABLE_SIZE. - """ - var p = Pointer(to=masks.unsafe_ptr()[unsafe_offset=Int(b) * cap]) - comptime if cap == NIBBLE_TABLE_SIZE: - return nibble_lookup( - p.unsafe_load[width=NIBBLE_TABLE_SIZE](), state_vec - ) - elif cap == 32: - return table_lookup_32(p.unsafe_load[width=32](), state_vec) - else: - comptime assert cap == 64 - return table_lookup_64(p.unsafe_load[width=64](), state_vec) - - -# Bytes walked between dead-state checks on the non-accelerated full-match -# loop. Bounds the wasted shuffles after an early death while keeping the -# vector->scalar state extract off the per-byte path. -comptime _SHENG_DEAD_CHECK_STRIDE = 64 - - def sheng_short_input(cap: Int) -> Int: """Comptime: input length below which an anchored full match walks the mask table scalar-wise (`_sheng_scalar_full_match`) instead of @@ -260,66 +220,6 @@ def _sheng_scalar_full_match[ return Int(cur) < d.num_match_states -@always_inline -def _sheng_full_match_impl[ - origin: Origin, - ns: Int, - //, - d: EagerDFA, - cap: Int, - masks: StringLiteral, - flags: Array[UInt8, ns], - accel: Bool, -](input: Span[Byte, origin]) -> Bool: - comptime dead = d.num_states - # `masks` / `flags` are comptime arrays; `materialize` binds them to the - # constant data emitted in the binary (no copy) so the walk can index them. - var flg = materialize[flags]() - var cur_vec = _ShuffleIndex(UInt8(d.start_at_0)) - var cur = d.start_at_0 - var pos = 0 - var input_len = len(input) - comptime if accel: - while pos < input_len: - var unused = -1 - var skipped = _edfa_accel_skip[d=d](input, cur, pos, unused) - comptime if _edfa_has_region(d): - var before = cur - skipped = _edfa_region_skip[d=d](input, cur, skipped) - if cur != before: - cur_vec = _ShuffleIndex(UInt8(cur)) - pos = skipped - if pos >= input_len: - break - cur_vec = _sheng_step[cap](masks, input.unsafe_get(pos), cur_vec) - cur = Int(cur_vec[0]) - if cur == dead: - return False - pos += 1 - else: - # The dead-state early exit is only an optimization (the dead - # state self-loops), so the vector->scalar extract runs once per - # stride instead of per byte — the per-byte loop is then just the - # load+shuffle dependency chain. - while pos < input_len: - var chunk_end = min(pos + _SHENG_DEAD_CHECK_STRIDE, input_len) - while pos < chunk_end: - cur_vec = _sheng_step[cap]( - masks, input.unsafe_get(pos), cur_vec - ) - pos += 1 - cur = Int(cur_vec[0]) - if cur == dead: - return False - comptime if d.any_eol_end: - return ( - cur < d.num_match_states - or (flg.unsafe_get(cur) & EDFA_EOL_AT_END) != 0 - ) - else: - return cur < d.num_match_states - - @always_inline def sheng_full_match[ origin: Origin, @@ -347,85 +247,14 @@ def sheng_full_match[ comptime if _edfa_has_accel(d): comptime W = simd_width_of[DType.uint8]() if len(input) >= W: - return _sheng_full_match_impl[ - d=d, cap=cap, masks=masks, flags=flags, accel=True + return _edfa_full_match_impl[ + d=d, table=masks, flags=flags, accel=True, cap=cap ](input) - return _sheng_full_match_impl[ - d=d, cap=cap, masks=masks, flags=flags, accel=False + return _edfa_full_match_impl[ + d=d, table=masks, flags=flags, accel=False, cap=cap ](input) -@always_inline -def _sheng_walk_impl[ - origin: Origin, - ns: Int, - //, - d: EagerDFA, - cap: Int, - masks: StringLiteral, - flags: Array[UInt8, ns], - accel: Bool, -](input: Span[Byte, origin], start: Int) -> Int: - comptime dead = d.num_states - var flg = materialize[flags]() - var cur: Int - if start == 0: - cur = d.start_at_0 - elif input.unsafe_get(start - 1) == CHAR_NEWLINE: - cur = d.start_after_nl - else: - comptime if d.start_other_word != d.start_other: - cur = d.start_other_word if is_word_byte( - input.unsafe_get(start - 1) - ) else d.start_other - else: - cur = d.start_other - var cur_vec = _ShuffleIndex(UInt8(cur)) - - var last_match = -1 - if cur < d.num_match_states: - last_match = start - - var pos = start - var input_len = len(input) - while pos < input_len: - comptime if accel: - var skipped = _edfa_accel_skip[d=d](input, cur, pos, last_match) - comptime if _edfa_has_region(d): - var before = cur - skipped = _edfa_region_skip[d=d](input, cur, skipped) - if cur != before: - cur_vec = _ShuffleIndex(UInt8(cur)) - pos = skipped - if pos >= input_len: - break - var b = input.unsafe_get(pos) - comptime if d.any_eol_nl: - if ( - b == CHAR_NEWLINE - and (flg.unsafe_get(cur) & EDFA_EOL_AT_NEWLINE) != 0 - ): - last_match = pos - comptime if d.any_wb: - if UInt(cur - d.num_match_states) < UInt(d.num_cond_states): - var f = flg.unsafe_get(cur) - if ((f & EDFA_MATCH_IF_WORD) != 0) == is_word_byte(b): - last_match = pos - cur_vec = _sheng_step[cap](masks, b, cur_vec) - cur = Int(cur_vec[0]) - if cur == dead: - # Died mid-input: EOL-at-end flags don't apply (mirrors - # edfa_match_at). - return last_match - pos += 1 - if cur < d.num_match_states: - last_match = pos - comptime if d.any_eol_end: - if (flg.unsafe_get(cur) & EDFA_EOL_AT_END) != 0: - last_match = pos - return last_match - - @always_inline def sheng_match_at[ origin: Origin, @@ -436,14 +265,5 @@ def sheng_match_at[ masks: StringLiteral, flags: Array[UInt8, ns], ](input: Span[Byte, origin], start: Int) -> Int: - """`edfa_match_at` on the shuffle engine, with the same per-walk - accelerated/plain dispatch.""" - comptime if _edfa_has_accel(d): - comptime W = simd_width_of[DType.uint8]() - if len(input) - start >= W: - return _sheng_walk_impl[ - d=d, cap=cap, masks=masks, flags=flags, accel=True - ](input, start) - return _sheng_walk_impl[ - d=d, cap=cap, masks=masks, flags=flags, accel=False - ](input, start) + """`edfa_match_at` on the shuffle engine.""" + return edfa_match_at[d=d, table=masks, flags=flags, cap=cap](input, start) diff --git a/emberregex/simd_kernels.mojo b/emberregex/simd_kernels.mojo index 3132045..08227a8 100644 --- a/emberregex/simd_kernels.mojo +++ b/emberregex/simd_kernels.mojo @@ -121,6 +121,31 @@ def table_lookup_64( ) +@always_inline +def _sheng_step[ + cap: Int +](masks: StringLiteral, b: Byte, state_vec: _ShuffleIndex) -> _ShuffleIndex: + """One Sheng transition (sheng.mojo): shuffle byte `b`'s `cap`-byte + mask by the state vector (the state id broadcast across the index + register; only lane 0 is ever read back, and its width is + independent of the mask width). + + Each branch loads and shuffles at the literal tier width `cap`: only + the tier this DFA needs is emitted, and the NEON-only tiers are never + elaborated where cap is always NIBBLE_TABLE_SIZE. + """ + var p = Pointer(to=masks.unsafe_ptr()[unsafe_offset=Int(b) * cap]) + comptime if cap == NIBBLE_TABLE_SIZE: + return nibble_lookup( + p.unsafe_load[width=NIBBLE_TABLE_SIZE](), state_vec + ) + elif cap == 32: + return table_lookup_32(p.unsafe_load[width=32](), state_vec) + else: + comptime assert cap == 64 + return table_lookup_64(p.unsafe_load[width=64](), state_vec) + + # --- Comptime mask builders ------------------------------------------------- diff --git a/emberregex/static_dfa.mojo b/emberregex/static_dfa.mojo index edf6fd9..43311ef 100644 --- a/emberregex/static_dfa.mojo +++ b/emberregex/static_dfa.mojo @@ -27,7 +27,9 @@ from .simd_kernels import ( ACCEL_SHUFTI, ACCEL_TRUFFLE, HAS_FAST_BYTE_SHUFFLE, + _ShuffleIndex, _class_contains, + _sheng_step, build_class_masks, build_shufti_masks, build_truffle_masks, @@ -2319,6 +2321,12 @@ def _pivot_forced_chain(d: EagerDFA, pv: Tuple[Int, Int]) -> List[Int]: return chain^ +# Bytes walked between dead-state checks on the plain Sheng full-match +# loop. Bounds the wasted shuffles after an early death while keeping the +# vector->scalar state extract off the per-byte path. +comptime _SHENG_DEAD_CHECK_STRIDE = 64 + + @always_inline def _edfa_full_match_impl[ origin: Origin, @@ -2328,28 +2336,65 @@ def _edfa_full_match_impl[ table: StringLiteral, flags: Array[UInt8, ns], accel: Bool, + cap: Int, ](input: Span[Byte, origin]) -> Bool: - # `table` / `flags` are comptime arrays; `materialize` binds them to the - # constant data emitted in the binary (no copy) so the walk can index them. + """Anchored full match. `cap == 0` steps the eager table (`table` from + `edfa_table_str`); a Sheng tier `cap` steps by one shuffle over the + mask table `table` (`sheng_masks_str`, sheng.mojo), dead state + `d.num_states`.""" + # `table` / `flags` are comptime constants; `materialize` binds them to + # the constant data emitted in the binary (no copy) so the walk can + # index them. comptime dt = edfa_id_dtype(d.num_states) var tbl = table.unsafe_ptr().unsafe_bitcast[Scalar[dt]]() var flg = materialize[flags]() var cur = d.start_at_0 + var cur_vec = _ShuffleIndex(UInt8(cur)) # Sheng state (cap > 0 only) var pos = 0 var input_len = len(input) - while pos < input_len: - comptime if accel: - var unused = -1 - pos = _edfa_accel_skip[d=d](input, cur, pos, unused) - comptime if _edfa_has_region(d): - pos = _edfa_region_skip[d=d](input, cur, pos) - if pos >= input_len: - break - var nxt = Int(tbl[unsafe_offset=cur * 256 + Int(input.unsafe_get(pos))]) - if nxt < 0: - return False - cur = nxt - pos += 1 + comptime if cap > 0 and not accel: + # The dead-state early exit is only an optimization (the dead + # state self-loops), so the vector->scalar extract runs once per + # stride instead of per byte — the per-byte loop is then just the + # load+shuffle dependency chain. + while pos < input_len: + var chunk_end = min(pos + _SHENG_DEAD_CHECK_STRIDE, input_len) + while pos < chunk_end: + cur_vec = _sheng_step[cap]( + table, input.unsafe_get(pos), cur_vec + ) + pos += 1 + cur = Int(cur_vec[0]) + if cur == d.num_states: + return False + else: + while pos < input_len: + comptime if accel: + var unused = -1 + pos = _edfa_accel_skip[d=d](input, cur, pos, unused) + comptime if _edfa_has_region(d): + var before = cur + pos = _edfa_region_skip[d=d](input, cur, pos) + comptime if cap > 0: + if cur != before: + cur_vec = _ShuffleIndex(UInt8(cur)) + if pos >= input_len: + break + comptime if cap > 0: + cur_vec = _sheng_step[cap]( + table, input.unsafe_get(pos), cur_vec + ) + cur = Int(cur_vec[0]) + if cur == d.num_states: + return False + else: + var nxt = Int( + tbl[unsafe_offset=cur * 256 + Int(input.unsafe_get(pos))] + ) + if nxt < 0: + return False + cur = nxt + pos += 1 comptime if d.any_eol_end: return ( cur < d.num_match_states @@ -2378,11 +2423,11 @@ def edfa_full_match[ comptime W = simd_width_of[DType.uint8]() if len(input) >= W: return _edfa_full_match_impl[ - d=d, table=table, flags=flags, accel=True + d=d, table=table, flags=flags, accel=True, cap=0 ](input) - return _edfa_full_match_impl[d=d, table=table, flags=flags, accel=False]( - input - ) + return _edfa_full_match_impl[ + d=d, table=table, flags=flags, accel=False, cap=0 + ](input) @always_inline @@ -2394,7 +2439,10 @@ def _edfa_walk_impl[ table: StringLiteral, flags: Array[UInt8, ns], accel: Bool, + cap: Int, ](input: Span[Byte, origin], start: Int) -> Int: + """The walk behind `edfa_match_at` / `sheng_match_at`; `cap` picks the + transition mechanism as in `_edfa_full_match_impl`.""" comptime dt = edfa_id_dtype(d.num_states) var tbl = table.unsafe_ptr().unsafe_bitcast[Scalar[dt]]() var flg = materialize[flags]() @@ -2410,6 +2458,7 @@ def _edfa_walk_impl[ ) else d.start_other else: cur = d.start_other + var cur_vec = _ShuffleIndex(UInt8(cur)) # Sheng state (cap > 0 only) var last_match = -1 if cur < d.num_match_states: @@ -2421,7 +2470,11 @@ def _edfa_walk_impl[ comptime if accel: pos = _edfa_accel_skip[d=d](input, cur, pos, last_match) comptime if _edfa_has_region(d): + var before = cur pos = _edfa_region_skip[d=d](input, cur, pos) + comptime if cap > 0: + if cur != before: + cur_vec = _ShuffleIndex(UInt8(cur)) if pos >= input_len: break var b = input.unsafe_get(pos) @@ -2439,12 +2492,18 @@ def _edfa_walk_impl[ var f = flg.unsafe_get(cur) if ((f & EDFA_MATCH_IF_WORD) != 0) == is_word_byte(b): last_match = pos - var nxt = Int(tbl[unsafe_offset=cur * 256 + Int(b)]) - if nxt < 0: - # Died mid-input: EOL-at-end flags don't apply (mirrors the - # `current >= 0` guard in LazyDFA.match_at). - return last_match - cur = nxt + # Died mid-input: EOL-at-end flags don't apply (mirrors the + # `current >= 0` guard in LazyDFA.match_at). + comptime if cap > 0: + cur_vec = _sheng_step[cap](table, b, cur_vec) + cur = Int(cur_vec[0]) + if cur == d.num_states: + return last_match + else: + var nxt = Int(tbl[unsafe_offset=cur * 256 + Int(b)]) + if nxt < 0: + return last_match + cur = nxt pos += 1 if cur < d.num_match_states: last_match = pos @@ -2462,6 +2521,7 @@ def edfa_match_at[ d: EagerDFA, table: StringLiteral, flags: Array[UInt8, ns], + cap: Int = 0, ](input: Span[Byte, origin], start: Int) -> Int: """Table walk from `start` in the DFA's own start states (position 0 / after '\n' / mid-line after a non-word byte / mid-line after a word @@ -2476,6 +2536,9 @@ def edfa_match_at[ ones, it is Python's leftmost-first end of the first match at or after `start`. + `cap > 0` walks the Sheng masks instead (`table` is then + `sheng_masks_str`; see `sheng_match_at`). + Dispatches once per walk between an accelerated and a plain loop: walks that can never reach a full vector chunk take the plain loop and pay no per-byte acceleration checks at all. @@ -2483,12 +2546,12 @@ def edfa_match_at[ comptime if _edfa_has_accel(d): comptime W = simd_width_of[DType.uint8]() if len(input) - start >= W: - return _edfa_walk_impl[d=d, table=table, flags=flags, accel=True]( - input, start - ) - return _edfa_walk_impl[d=d, table=table, flags=flags, accel=False]( - input, start - ) + return _edfa_walk_impl[ + d=d, table=table, flags=flags, accel=True, cap=cap + ](input, start) + return _edfa_walk_impl[ + d=d, table=table, flags=flags, accel=False, cap=cap + ](input, start) @always_inline From c8da766a2301e76c25ec580c738b25720ac84aae Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:07:44 -0600 Subject: [PATCH 24/65] cleanup: set_oracle drops the region-bounded sweep/sweep_som sweep_ctx superseded them (MULTIPATTERN_PLAN.md: the region-bounded oracle was unsound for lookahead and anchors). __main__ now runs sweep_ctx; _pin keeps a leading global flag group ((?i), (?s)) outside its wrapper, which Python otherwise rejects. sweep_ctx_som stays as the SOM oracle and takes over sweep_som's doc references. Case-table output is byte-identical, and sweep_ctx_som matches the old sweep_som on every case. Co-Authored-By: Claude Opus 5.5 (1M context) --- ARCHITECTURE.md | 6 ++-- MULTIPATTERN_PLAN.md | 3 +- test/test_set_phase5.mojo | 4 +-- test/test_set_phase7.mojo | 2 +- tools/set_oracle.py | 61 +++++++++------------------------------ 5 files changed, 21 insertions(+), 55 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3f28c34..5432552 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -573,9 +573,9 @@ Three constraints shape the code more than anything else: Every engine is differentially tested against the tagged Pike reference across LCG-generated inputs at chunk-boundary-adjacent lengths, including bytes ≥ 0x80. Set semantics are ground-truthed against CPython via -`tools/set_oracle.py` — including `sweep_ctx`, a context-preserving variant -that is sound for anchors and lookaround where the naive region-bounded -sweep is not. Streaming is checked by exhaustive block/stream equivalence +`tools/set_oracle.py` — `sweep_ctx`, a context-preserving sweep that is +sound for anchors and lookaround where a naive region-bounded sweep is +not. Streaming is checked by exhaustive block/stream equivalence over every 2- and 3-way chunk split. Benches live in `bench/bench.mojo` (single pattern) and diff --git a/MULTIPATTERN_PLAN.md b/MULTIPATTERN_PLAN.md index f40194c..71947f3 100644 --- a/MULTIPATTERN_PLAN.md +++ b/MULTIPATTERN_PLAN.md @@ -37,7 +37,8 @@ list at the end is deliberate and specific, not a summary of intent. slots give leftmost SOM for free from the generation counter. `scan_spans` filters the stream to per-id leftmost non-overlapping spans. Verified by differentials between the two independent - implementations plus `tools/set_oracle.py::sweep_som`. + implementations plus `tools/set_oracle.py::sweep_som` (since replaced + by the context-preserving `sweep_ctx_som`). **Not done: SOM horizon modes (5.3)** — they are an offset-WIDTH tradeoff in stream state, and our offsets are plain `Int`, so there is nothing to trade until stream-state size is itself a problem. diff --git a/test/test_set_phase5.mojo b/test/test_set_phase5.mojo index cc38d8a..2fe5833 100644 --- a/test/test_set_phase5.mojo +++ b/test/test_set_phase5.mojo @@ -5,8 +5,8 @@ of any match of `id` ending at `end`. Two independent implementations back it — a determinized reverse automaton walked leftward, and per-thread start slots in the Pike VM — so most of the confidence here comes from differentials between them, on top of expectations derived -from CPython (`tools/set_oracle.py::sweep_som`, the O(n²) sweep, which is -sound only for anchor-free patterns; anchored cases are hand-derived). +from CPython (`tools/set_oracle.py::sweep_ctx_som`, the O(n²) sweep; +anchored cases here are hand-derived). `scan_spans` filters that stream to per-id leftmost non-overlapping spans. It is leftmost-LONGEST, not CPython's leftmost-first; the tests diff --git a/test/test_set_phase7.mojo b/test/test_set_phase7.mojo index 224a427..386cff6 100644 --- a/test/test_set_phase7.mojo +++ b/test/test_set_phase7.mojo @@ -156,7 +156,7 @@ def test_keyword_parameter_construction() raises: # --- Exact backreferences and lookaround (the Hyperscan differentiator) ----- # # Expectations come from tools/set_oracle.py::sweep_ctx, the -# CONTEXT-PRESERVING all-ends sweep. The plain `sweep` is unsound here for +# CONTEXT-PRESERVING all-ends sweep. A region-bounded sweep is unsound for # the same reason a first cut of the engine was wrong: bounding the region # hides the right-hand text a lookahead asserts about. diff --git a/tools/set_oracle.py b/tools/set_oracle.py index 3c1ffb6..99b7a1a 100644 --- a/tools/set_oracle.py +++ b/tools/set_oracle.py @@ -6,13 +6,9 @@ All-ends semantics CANNOT be derived from re.finditer (`ab|a` on "ab" must report end 1 AND end 2). The sound oracle is the O(n^2) sweep: -end p is reportable for pattern i iff re.fullmatch(pat_i, input[s:p]) -for some s <= p. - -VALIDITY LIMIT: slicing breaks ^/$/\b/\B/(?m) relative anchoring, so -the sweep is sound only for patterns free of anchors and word -boundaries. Anchored cases must be hand-derived (see -test/test_set_phase0.mojo). This script refuses anchored patterns. +end p is reportable for pattern i iff pattern i matches exactly +input[s:p] for some s <= p, with the whole input still visible (`_pin`), +so anchors, word boundaries and lookaround see their real context. Everything runs on BYTES so offsets match emberregex's byte-mode semantics (Python's bytes regexes are ASCII-only for \d/\w/\s, same as @@ -24,43 +20,7 @@ import re -ANCHOR_RE = re.compile(r"(? list[tuple[int, int]]: - for p in patterns: - if ANCHOR_RE.search(p.decode("latin-1")): - raise ValueError(f"anchored pattern {p!r}: sweep oracle unsound") - compiled = [re.compile(p) for p in patterns] - n = len(data) - out = [] - for end in range(n + 1): - for i, c in enumerate(compiled): - if any(c.fullmatch(data, s, end) for s in range(end + 1)): - out.append((i, end)) - return out - - -def sweep_som(patterns: list[bytes], data: bytes) -> list[tuple[int, int, int]]: - """Same sweep, additionally reporting the LEFTMOST start per (id, end). - - Contract check for `scan_som` (MULTIPATTERN_PLAN.md phase 5): the - reported start is the smallest s for which pattern i fullmatches - data[s:end]. Same anchor validity limit as `sweep`. - """ - for p in patterns: - if ANCHOR_RE.search(p.decode("latin-1")): - raise ValueError(f"anchored pattern {p!r}: sweep oracle unsound") - compiled = [re.compile(p) for p in patterns] - n = len(data) - out = [] - for end in range(n + 1): - for i, c in enumerate(compiled): - for s in range(end + 1): - if c.fullmatch(data, s, end): - out.append((i, s, end)) - break - return out +_GLOBAL_FLAGS = re.compile(rb"\(\?[aiLmsux]+\)") def _pin(pattern: bytes, remaining: int) -> bytes: @@ -71,9 +31,13 @@ def _pin(pattern: bytes, remaining: int) -> bytes: rest of the string: a lookahead then asserts against a truncated view and `$` fires early. The trailing lookahead below pins the end while leaving the whole string visible, which is what the engine - itself does (`end_at` in backtrack.mojo). + itself does (`end_at` in backtrack.mojo). A leading global flag group + (`(?i)`, `(?s)`) stays in front: Python rejects one inside the wrapper. """ - return b"(?:" + pattern + b")(?=[\\s\\S]{" + str(remaining).encode() + b"}\\Z)" + flags = _GLOBAL_FLAGS.match(pattern) + lead = flags.group(0) if flags else b"" + body = pattern[len(lead):] + return lead + b"(?:" + body + b")(?=[\\s\\S]{" + str(remaining).encode() + b"}\\Z)" def sweep_ctx(patterns: list[bytes], data: bytes) -> list[tuple[int, int]]: @@ -92,7 +56,8 @@ def sweep_ctx(patterns: list[bytes], data: bytes) -> list[tuple[int, int]]: def sweep_ctx_som( patterns: list[bytes], data: bytes ) -> list[tuple[int, int, int]]: - """`sweep_ctx` additionally reporting the leftmost start per (id, end).""" + """`sweep_ctx` additionally reporting the leftmost start per (id, end): + the contract check for `scan_som` (MULTIPATTERN_PLAN.md phase 5).""" n = len(data) out = [] for end in range(n + 1): @@ -185,7 +150,7 @@ def fmt_mojo(reports: list[tuple[int, int]]) -> str: if __name__ == "__main__": for name, pats, data in CASES: - r = sweep(pats, data) + r = sweep_ctx(pats, data) print(f"# {name}: patterns={pats} input={data!r}") print(f"# {len(r)} reports") print(fmt_mojo(r)) From c4110e132a21dd09530a73700c8eb4c13b084805 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:07:54 -0600 Subject: [PATCH 25/65] cleanup: share the literal-alternation head expansion and chain walk Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/optimize.mojo | 181 ++++++++++++++++----------------------- 1 file changed, 73 insertions(+), 108 deletions(-) diff --git a/emberregex/optimize.mojo b/emberregex/optimize.mojo index 8baf7f1..d973342 100644 --- a/emberregex/optimize.mojo +++ b/emberregex/optimize.mojo @@ -168,30 +168,23 @@ struct LiteralAlt(Copyable, Movable): self.min_len = 0 -def extract_literal_alternation(nfa: NFA) -> LiteralAlt: - """Comptime: detect a pure alternation of 2..TEDDY_MAX_LITERALS plain - literals. - - The start must expand (through no-op SPLITs and SAVEs) into a SPLIT - tree whose leaves are CHAR chains ending at MATCH. Anything else — - anchors, charsets, quantifier cycles, empty branches, nested - alternation mid-chain — invalidates the extraction. - """ - var result = LiteralAlt() +def _alt_heads(nfa: NFA) -> List[Int]: + """Comptime: the branch heads of the SPLIT tree the start expands into + (through no-op SPLITs and SAVEs), when every leaf is a CHAR or a + filterable CHARSET and there are 2..TEDDY_MAX_LITERALS of them; empty + otherwise. The expansion budget rejects quantifier cycles (which + revisit SPLITs indefinitely).""" var num_states = len(nfa.states) - - # Expand the alternation tree into branch heads. The expansion budget - # rejects quantifier cycles (which revisit SPLITs indefinitely). var heads = List[Int]() var stack: List[Int] = [nfa.start] var budget = 4 * TEDDY_MAX_LITERALS while len(stack) > 0: budget -= 1 if budget < 0: - return result^ + return List[Int]() var s = stack.pop() if s < 0 or s >= num_states: - return result^ + return List[Int]() var kind = nfa.states[s].kind if kind == NFAStateKind.SPLIT: if nfa.states[s].out2 == -1: @@ -209,44 +202,74 @@ def extract_literal_alternation(nfa: NFA) -> LiteralAlt: ): heads.append(s) # (?i) case pair or single-member charset else: - return result^ + return List[Int]() if len(heads) < 2 or len(heads) > TEDDY_MAX_LITERALS: + return List[Int]() + return heads^ + + +def _lit_chain( + nfa: NFA, head: Int, cap: Int, mut bytes: List[Int], mut cl: List[Bool] +) -> Bool: + """Comptime: append to `bytes` (and the parallel caseless flags `cl`) + the literal bytes of the CHAR / filterable-CHARSET chain from `head`, + through SAVEs and no-op SPLITs, stopping at the first other state or + at `cap` bytes. True when the chain stopped at MATCH.""" + var num_states = len(nfa.states) + var s = head + var steps = 0 + var ended_at_match = False + while len(bytes) < cap: + steps += 1 + if steps > num_states or s < 0 or s >= num_states: + break + var kind = nfa.states[s].kind + if kind == NFAStateKind.CHAR: + var cv = nfa.states[s].char_value + if cv >= 256: + break + bytes.append(Int(cv)) + cl.append(False) + s = nfa.states[s].out1 + elif kind == NFAStateKind.CHARSET: + var fb = _charset_filter_byte(nfa, nfa.states[s].charset_index) + if fb[0] < 0: + break # unfilterable charset ends the chain + bytes.append(fb[0]) + cl.append(fb[1]) + s = nfa.states[s].out1 + elif kind == NFAStateKind.SAVE: + s = nfa.states[s].out1 + elif kind == NFAStateKind.SPLIT and nfa.states[s].out2 == -1: + s = nfa.states[s].out1 + else: + ended_at_match = kind == NFAStateKind.MATCH + break + return ended_at_match + + +def extract_literal_alternation(nfa: NFA) -> LiteralAlt: + """Comptime: detect a pure alternation of 2..TEDDY_MAX_LITERALS plain + literals. + + The start must expand (through no-op SPLITs and SAVEs) into a SPLIT + tree whose leaves are CHAR chains ending at MATCH. Anything else — + anchors, charsets, quantifier cycles, empty branches, nested + alternation mid-chain — invalidates the extraction. + """ + var result = LiteralAlt() + var num_states = len(nfa.states) + var heads = _alt_heads(nfa) + if len(heads) == 0: return result^ var min_len = num_states # any literal is shorter than the NFA for h in heads: + # A chain has fewer bytes than the NFA has states, so this cap + # never cuts one short of MATCH. var bytes = List[Int]() var cl = List[Bool]() - var s = h - var steps = 0 - while True: - steps += 1 - if steps > num_states or s < 0 or s >= num_states: - return result^ - var kind = nfa.states[s].kind - if kind == NFAStateKind.CHAR: - var cv = nfa.states[s].char_value - if cv >= 256: - return result^ - bytes.append(Int(cv)) - cl.append(False) - s = nfa.states[s].out1 - elif kind == NFAStateKind.CHARSET: - var fb = _charset_filter_byte(nfa, nfa.states[s].charset_index) - if fb[0] < 0: - return result^ - bytes.append(fb[0]) - cl.append(fb[1]) - s = nfa.states[s].out1 - elif kind == NFAStateKind.SAVE: - s = nfa.states[s].out1 - elif kind == NFAStateKind.SPLIT and nfa.states[s].out2 == -1: - s = nfa.states[s].out1 - elif kind == NFAStateKind.MATCH: - break - else: - return result^ - if len(bytes) == 0: + if not _lit_chain(nfa, h, num_states, bytes, cl) or len(bytes) == 0: return result^ if len(bytes) < min_len: min_len = len(bytes) @@ -270,37 +293,8 @@ def extract_alt_prefix(nfa: NFA) -> LiteralAlt: full Teddy engine owns that), when any arm starts with a non-CHAR state, or when more than TEDDY_MAX_LITERALS arms exist.""" var result = LiteralAlt() - var num_states = len(nfa.states) - - var heads = List[Int]() - var stack: List[Int] = [nfa.start] - var budget = 4 * TEDDY_MAX_LITERALS - while len(stack) > 0: - budget -= 1 - if budget < 0: - return result^ - var s = stack.pop() - if s < 0 or s >= num_states: - return result^ - var kind = nfa.states[s].kind - if kind == NFAStateKind.SPLIT: - if nfa.states[s].out2 == -1: - stack.append(nfa.states[s].out1) - else: - stack.append(nfa.states[s].out2) - stack.append(nfa.states[s].out1) - elif kind == NFAStateKind.SAVE: - stack.append(nfa.states[s].out1) - elif kind == NFAStateKind.CHAR: - heads.append(s) - elif ( - kind == NFAStateKind.CHARSET - and _charset_filter_byte(nfa, nfa.states[s].charset_index)[0] >= 0 - ): - heads.append(s) # (?i) case pair or single-member charset - else: - return result^ - if len(heads) < 2 or len(heads) > TEDDY_MAX_LITERALS: + var heads = _alt_heads(nfa) + if len(heads) == 0: return result^ comptime CHAIN_CAP = 8 # verification cost bound per candidate @@ -309,39 +303,10 @@ def extract_alt_prefix(nfa: NFA) -> LiteralAlt: for h in heads: var bytes = List[Int]() var cl = List[Bool]() - var s = h - var steps = 0 - var ended_at_match = False - while len(bytes) < CHAIN_CAP: - steps += 1 - if steps > num_states or s < 0 or s >= num_states: - break - var kind = nfa.states[s].kind - if kind == NFAStateKind.CHAR: - var cv = nfa.states[s].char_value - if cv >= 256: - break - bytes.append(Int(cv)) - cl.append(False) - s = nfa.states[s].out1 - elif kind == NFAStateKind.CHARSET: - var fb = _charset_filter_byte(nfa, nfa.states[s].charset_index) - if fb[0] < 0: - break # unfilterable charset ends the chain - bytes.append(fb[0]) - cl.append(fb[1]) - s = nfa.states[s].out1 - elif kind == NFAStateKind.SAVE: - s = nfa.states[s].out1 - elif kind == NFAStateKind.SPLIT and nfa.states[s].out2 == -1: - s = nfa.states[s].out1 - else: - if kind == NFAStateKind.MATCH: - ended_at_match = True - break + var at_match = _lit_chain(nfa, h, CHAIN_CAP, bytes, cl) if len(bytes) < 2: return result^ # a 1-byte arm filters no better than the bitmap - if not ended_at_match: + if not at_match: all_end_at_match = False if len(bytes) < min_len: min_len = len(bytes) From 8a2bb7760bc91a2df785ca2d0b75aa83a203aeaa Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:08:05 -0600 Subject: [PATCH 26/65] cleanup: LazyDFA.search_forward reuses match_at's walk The per-start walk moves into an @always_inline _longest_end that both match_at and search_forward inline, so neither gains a call. The unreachable pos > input_len break goes with it. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/dfa.mojo | 43 +++++++++---------------------------------- 1 file changed, 9 insertions(+), 34 deletions(-) diff --git a/emberregex/dfa.mojo b/emberregex/dfa.mojo index a6af705..947e84f 100644 --- a/emberregex/dfa.mojo +++ b/emberregex/dfa.mojo @@ -262,6 +262,14 @@ struct LazyDFA(Copyable, Movable): comptime tables overflowed. """ self._ensure_init(nfa) + return self._longest_end(nfa, input, start) + + @always_inline + def _longest_end[ + origin: Origin, // + ](mut self, nfa: NFA, input: Span[Byte, origin], start: Int) raises -> Int: + """`match_at`'s walk, without the init check: inlined into it and + into `search_forward`'s per-start loop.""" var input_len = len(input) # Select initial state based on position context @@ -332,40 +340,7 @@ struct LazyDFA(Copyable, Movable): break pos += 1 - if pos > input_len: - break - - # Select initial state based on position context - var current: Int - if pos == 0: - current = self._init_start - elif pos > 0 and input.unsafe_get(pos - 1) == CHAR_NEWLINE: - current = self._init_after_nl - else: - current = self._init_other - - var last_match = -1 - if self.states.unsafe_get(current).is_match: - last_match = pos - - self._begin_run(pos) - var p = pos - while p < input_len: - var byte = input.unsafe_get(p) - if byte == CHAR_NEWLINE: - if self.states.unsafe_get(current).eol_at_newline: - last_match = p - current = self._step(nfa, current, byte, p) - if current < 0: - break - p += 1 - if self.states.unsafe_get(current).is_match: - last_match = p - self._end_run(p) - - if current >= 0 and self.states.unsafe_get(current).eol_at_end: - last_match = p - + var last_match = self._longest_end(nfa, input, pos) if last_match >= 0: return (pos, last_match) From 26c92e2b7cb248ae1a0403a54b28fca5d2c33e5f Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:08:12 -0600 Subject: [PATCH 27/65] cleanup: hyperscan compare imports PAIRS/run_mojo from bench_compare_set The same bench_set table parse and the same nine row pairs lived in both scripts; the haystack size now comes from the row name's _16k/_64k suffix. A failing `pixi run bench_set` now prints an error and yields no rows (bench_compare_set's handling) instead of a traceback. Co-Authored-By: Claude Opus 5.5 (1M context) --- bench/bench_compare_hyperscan.py | 39 +++++--------------------------- 1 file changed, 6 insertions(+), 33 deletions(-) diff --git a/bench/bench_compare_hyperscan.py b/bench/bench_compare_hyperscan.py index 78379d2..e5d066c 100644 --- a/bench/bench_compare_hyperscan.py +++ b/bench/bench_compare_hyperscan.py @@ -32,26 +32,14 @@ import subprocess import sys +from bench_compare_set import PAIRS, run_mojo + ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) HS_ENV = os.path.join(ROOT, ".pixi", "envs", "hs") SRC = os.path.join(ROOT, "comparisons", "bench_hyperscan.c") BIN = os.path.join(ROOT, "comparisons", "bench_hyperscan") COUNTS = os.path.join(ROOT, "comparisons", "set_counts.mojo") -# Mojo bench row -> the C harness row it should be read against, and the -# haystack size in bytes (for GB/s). -PAIRS = [ - ("set_teddy8_sparse_16k", "teddy8_sparse_16k", 16 * 1024), - ("set_teddy8_dense_16k", "teddy8_dense_16k", 16 * 1024), - ("set_teddy64_sparse_16k", "teddy64_sparse_16k", 16 * 1024), - ("set_teddy64_dense_16k", "teddy64_dense_16k", 16 * 1024), - ("set_rose_log_sparse_16k", "log_sparse_16k", 16 * 1024), - ("set_rose_log_dense_16k", "log_dense_16k", 16 * 1024), - ("set_rose_log_sparse_64k", "log_sparse_64k", 64 * 1024), - ("set_rose_full_sparse_64k", "full_sparse_64k", 64 * 1024), - ("set_rose_full_dense_16k", "full_dense_16k", 16 * 1024), -] - def build(): inc = os.path.join(HS_ENV, "include") @@ -106,24 +94,6 @@ def run_mojo_counts(): return rows -def run_mojo(): - """{row: GB/s}""" - out = subprocess.run( - ["pixi", "run", "bench_set"], capture_output=True, text=True, check=True - ).stdout - rows = {} - for line in out.splitlines(): - if not line.startswith("| set_"): - continue - cells = [c.strip() for c in line.split("|")] - if len(cells) > 4: - try: - rows[cells[1]] = float(cells[4]) - except ValueError: - pass - return rows - - def main(): print("building the Vectorscan harness...") build() @@ -138,7 +108,10 @@ def main(): print(f"{'row':<28} {'ember':>9} {'vectorscan':>11} {'ratio':>8} matches") print("-" * 74) mismatched = 0 - for mojo_row, hs_row, nbytes in PAIRS: + # The C harness names its rows like the Python baselines (PAIRS' second + # column), and every row name ends in its haystack size ("..._16k"). + for mojo_row, hs_row in PAIRS: + nbytes = int(hs_row.rsplit("_", 1)[1].rstrip("k")) * 1024 e = mo.get(mojo_row) h = hs.get(hs_row) if e is None or h is None: From a9b2d517c6250912a2940da8656a85acfe134b9b Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:08:59 -0600 Subject: [PATCH 28/65] cleanup: run_test.py drops the bench skip and two one-line helpers `if "bench" in root` matched the absolute walk root, so a checkout under any path containing "bench" skipped every test (test/ has no bench dir). needs_pkg_rebuild (a != wrapper) and prune_results (one dict comprehension, one caller) are inlined; their unit tests go with them. Co-Authored-By: Claude Opus 5.5 (1M context) --- run_test.py | 17 +++-------------- tools/test_run_test.py | 32 -------------------------------- 2 files changed, 3 insertions(+), 46 deletions(-) diff --git a/run_test.py b/run_test.py index 4eef1e8..5eb2ada 100755 --- a/run_test.py +++ b/run_test.py @@ -186,16 +186,6 @@ def wall(p): return sorted(paths, key=lambda p: (-wall(p), p))[:n] -def prune_results(results, live_paths): - """Drop recorded entries for files no longer present, so the store - does not accumulate stale durations for deleted tests.""" - return {p: v for p, v in results.items() if p in live_paths} - - -def needs_pkg_rebuild(stored_fp, current_fp): - return stored_fp != current_fp - - # --- File collection -------------------------------------------------------- @@ -207,8 +197,6 @@ def collect_files(test_dir="test"): every record and exit green.""" normal, cfail = [], [] for root, _, files in sorted(os.walk(os.path.join(ROOT, test_dir))): - if "bench" in root: - continue for file in sorted(files): if file.endswith(".mojo"): path = os.path.relpath(os.path.join(root, file), ROOT) @@ -243,7 +231,7 @@ def ensure_package(fp): stored = json.load(f).get("fingerprint") except (json.JSONDecodeError, OSError): stored = None - if not needs_pkg_rebuild(stored, fp): + if stored == fp: return True os.makedirs(CACHE_DIR, exist_ok=True) t0 = time.monotonic() @@ -496,7 +484,8 @@ def store(path, kind, ok, dur, tests): # filtered (--only) run has not looked at the other files and must # not evict their records. if not args.only: - results = prune_results(results, set(normal) | set(cfail)) + live = set(normal) | set(cfail) + results = {p: v for p, v in results.items() if p in live} save_results(results) if skipped: diff --git a/tools/test_run_test.py b/tools/test_run_test.py index 8572e61..2162f81 100644 --- a/tools/test_run_test.py +++ b/tools/test_run_test.py @@ -191,38 +191,6 @@ def test_passing_files_unaffected_when_none_failed(self): self.assertEqual(order, ["test/b.mojo", "test/a.mojo"]) -class TestPruneResults(unittest.TestCase): - def test_drops_entries_for_files_no_longer_present(self): - results = { - "test/live.mojo": {"duration": 1.0, "status": "pass"}, - "test/deleted.mojo": {"duration": 2.0, "status": "pass"}, - } - pruned = run_test.prune_results(results, {"test/live.mojo"}) - self.assertIn("test/live.mojo", pruned) - self.assertNotIn("test/deleted.mojo", pruned) - - def test_keeps_all_live_entries(self): - results = { - "test/a.mojo": {"duration": 1.0, "status": "pass"}, - "test/b.mojo": {"duration": 2.0, "status": "fail"}, - } - pruned = run_test.prune_results( - results, {"test/a.mojo", "test/b.mojo"} - ) - self.assertEqual(pruned, results) - - -class TestPkgRebuild(unittest.TestCase): - def test_missing_record_rebuilds(self): - self.assertTrue(run_test.needs_pkg_rebuild(None, "fp")) - - def test_stale_fingerprint_rebuilds(self): - self.assertTrue(run_test.needs_pkg_rebuild("old", "fp")) - - def test_fresh_fingerprint_skips_rebuild(self): - self.assertFalse(run_test.needs_pkg_rebuild("fp", "fp")) - - class TestDurationEstimate(unittest.TestCase): def test_warm_run_does_not_demote_a_cold_heavy_file(self): # The scheduling estimate is the larger of this run's wall time From 4a0a4ddd0fa56c715d2714253d757e19df646d9a Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:09:13 -0600 Subject: [PATCH 29/65] cleanup: one required-byte fast-fail helper, one lazy-DFA except comment Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/engine.mojo | 86 +++++++++++++----------------------------- 1 file changed, 26 insertions(+), 60 deletions(-) diff --git a/emberregex/engine.mojo b/emberregex/engine.mojo index 97bce1b..fbbf2ad 100644 --- a/emberregex/engine.mojo +++ b/emberregex/engine.mojo @@ -2302,16 +2302,8 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( def search(mut self, input: String) -> MatchResult[Self._num_slots]: """Search for the first occurrence of the pattern in the input.""" - comptime if Self._strategy.required_byte >= 0: - if ( - simd_find_byte( - input.as_bytes(), - UInt8(Self._strategy.required_byte), - 0, - ) - < 0 - ): - return MatchResult[Self._num_slots].no_match() + if self._required_byte_absent(input): + return MatchResult[Self._num_slots].no_match() comptime if Self._strategy.use_simd_literal: var lit = rebind[TypeForPrefixLength[Self._strategy.prefix_len]]( self._simd_lit @@ -2401,12 +2393,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( return MatchResult[Self._num_slots].no_match() return MatchResult[Self._num_slots].no_match() except: - # Only the lazy DFA can raise here (DFA_STATE_CAP): for - # eager/Sheng/Teddy tables the handler is dead, yet an - # unreachable `except` body still ELABORATES, and naming - # `_pike_*` drags the runtime parser + NFA builder + Pike - # VM into every binary. Gate the body on the lane that can - # actually raise. + # See match(): only the lazy DFA can raise here. comptime if Self._use_lazy_dfa: return self._pike_search(input) else: @@ -2553,16 +2540,8 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( No per-match String allocation: slice lazily via span() / group_str(). findall() is a wrapper over this.""" - comptime if Self._strategy.required_byte >= 0: - if ( - simd_find_byte( - input.as_bytes(), - UInt8(Self._strategy.required_byte), - 0, - ) - < 0 - ): - return List[MatchResult[Self._num_slots]]() + if self._required_byte_absent(input): + return List[MatchResult[Self._num_slots]]() comptime if Self._strategy.use_simd_literal: var lit = rebind[TypeForPrefixLength[Self._strategy.prefix_len]]( self._simd_lit @@ -2697,12 +2676,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( ) return results^ except: - # Only the lazy DFA can raise here (DFA_STATE_CAP): for - # eager/Sheng/Teddy tables the handler is dead, yet an - # unreachable `except` body still ELABORATES, and naming - # `_pike_*` drags the runtime parser + NFA builder + Pike - # VM into every binary. Gate the body on the lane that can - # actually raise. + # See match(): only the lazy DFA can raise here. comptime if Self._use_lazy_dfa: return self._pike_finditer(input) else: @@ -2731,16 +2705,8 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( wrapper over it: materializing the intermediate MatchResult list measured 1.3-1.9x on findall-heavy rows. Keep the iteration structure of the two in sync.""" - comptime if Self._strategy.required_byte >= 0: - if ( - simd_find_byte( - input.as_bytes(), - UInt8(Self._strategy.required_byte), - 0, - ) - < 0 - ): - return List[String]() + if self._required_byte_absent(input): + return List[String]() comptime if Self._strategy.use_simd_literal: var lit = rebind[TypeForPrefixLength[Self._strategy.prefix_len]]( self._simd_lit @@ -2890,12 +2856,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( ) return results^ except: - # Only the lazy DFA can raise here (DFA_STATE_CAP): for - # eager/Sheng/Teddy tables the handler is dead, yet an - # unreachable `except` body still ELABORATES, and naming - # `_pike_*` drags the runtime parser + NFA builder + Pike - # VM into every binary. Gate the body on the lane that can - # actually raise. + # See match(): only the lazy DFA can raise here. comptime if Self._use_lazy_dfa: return self._pike_findall(input) else: @@ -3183,12 +3144,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( try: return self._replace_dfa(input, replacement) except: - # Only the lazy DFA can raise here (DFA_STATE_CAP): for - # eager/Sheng/Teddy tables the handler is dead, yet an - # unreachable `except` body still ELABORATES, and naming - # `_pike_*` drags the runtime parser + NFA builder + Pike - # VM into every binary. Gate the body on the lane that can - # actually raise. + # See match(): only the lazy DFA can raise here. comptime if Self._use_lazy_dfa: return self._pike_replace(input, replacement) else: @@ -3460,12 +3416,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( input_bytes, start ) except: - # Only the lazy DFA can raise here (DFA_STATE_CAP): for - # eager/Sheng/Teddy tables the handler is dead, yet an - # unreachable `except` body still ELABORATES, and naming - # `_pike_*` drags the runtime parser + NFA builder + Pike - # VM into every binary. Gate the body on the lane that can - # actually raise. + # See match(): only the lazy DFA can raise here. comptime if Self._use_lazy_dfa: return self._pike_split(input) else: @@ -3597,6 +3548,21 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( off_b=probes[1], ](input, start) + @always_inline + def _required_byte_absent(self, input: String) -> Bool: + """Required-byte fast-fail for the unanchored verbs (see + `_compute_strategy`): True when a byte every match contains is + missing from `input`. Always False, at comptime, without one.""" + comptime if Self._strategy.required_byte >= 0: + return ( + simd_find_byte( + input.as_bytes(), UInt8(Self._strategy.required_byte), 0 + ) + < 0 + ) + else: + return False + @always_inline def _first_byte_hit(self, b: Byte) -> Bool: """Is `b` in the pattern's first-byte set? One bit of From 9bbc864c97796300ecedbacbdfc264452c4f804c Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:09:41 -0600 Subject: [PATCH 30/65] cleanup: drop bench/tooling leftovers and the unused pixi build task - bench_compare: run_mojo_static_benchmarks absorbs _run_mojo_task (its only caller); comments naming the nonexistent bench_static.mojo now name bench.mojo. - unused: statistics (comptime_stages), TA_CENTER and LIGHT_GREEN (gen_pdf). - pixi.toml: the `build` task (mojo precompile -> emberregex.mojoc) was referenced nowhere, duplicated run_test.ensure_package and read like `pixi build`. The lock is unaffected (pixi lock --dry-run --check). Co-Authored-By: Claude Opus 5.5 (1M context) --- bench/bench_compare.py | 17 ++++++----------- bench/bench_compare_pcre2.py | 2 +- bench/gen_pdf.py | 2 -- pixi.toml | 1 - tools/comptime_stages.py | 3 +-- 5 files changed, 8 insertions(+), 17 deletions(-) diff --git a/bench/bench_compare.py b/bench/bench_compare.py index f2a3971..f881935 100644 --- a/bench/bench_compare.py +++ b/bench/bench_compare.py @@ -22,7 +22,7 @@ NUMBER = 10000 # calls per repetition (sum of many runs → less variance) BAR_COLS = 20 # width of the speedup bar -# Must match comptime ITERS_PER_CALL in bench_static.mojo. +# Must match comptime ITERS_PER_CALL in bench.mojo. # Each Mojo call() invocation runs the function this many times; divide to get per-call µs. MOJO_ITERS_PER_CALL = 100 @@ -44,15 +44,15 @@ def section(title): print(f"{'─'*72}") -def _run_mojo_task(task: str) -> dict[str, float]: - """Run a pixi bench task and parse its markdown table output.""" +def run_mojo_static_benchmarks() -> dict[str, float]: + """Run `pixi run bench` (bench/bench.mojo) and parse its markdown table.""" pixi_cmd = shutil.which("pixi") if pixi_cmd is None: - print(f" [warning] pixi not found in PATH — skipping {task}") + print(" [warning] pixi not found in PATH — skipping bench") return {} result = subprocess.run( - [pixi_cmd, "run", task], + [pixi_cmd, "run", "bench"], capture_output=True, text=True, cwd=os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."), ) @@ -83,11 +83,6 @@ def _run_mojo_task(task: str) -> dict[str, float]: return timings -def run_mojo_static_benchmarks() -> dict[str, float]: - """Run bench_static.mojo (Regex) via pixi.""" - return _run_mojo_task("bench") - - def speedup_bar(ratio: float, cols: int = BAR_COLS) -> str: """Return a coloured bar string representing the speedup ratio.""" filled = min(int(ratio / 10.0 * cols), cols) if ratio <= 10 else cols @@ -167,7 +162,7 @@ def print_comparison( # --------------------------------------------------------------------------- -# Python re benchmark suite (names must match bench_static.mojo BenchIds) +# Python re benchmark suite (names must match bench.mojo BenchIds) # --------------------------------------------------------------------------- def run_python_benchmarks() -> dict[str, float]: diff --git a/bench/bench_compare_pcre2.py b/bench/bench_compare_pcre2.py index 2cc2320..d5f4f6c 100644 --- a/bench/bench_compare_pcre2.py +++ b/bench/bench_compare_pcre2.py @@ -157,7 +157,7 @@ def main() -> None: if not pcre2: print("\n [note] PCRE2 data unavailable.") if not static: - print("\n [note] Regex data unavailable (pixi run bench_static failed).") + print("\n [note] Regex data unavailable (pixi run bench failed).") if args.pdf: if "reportlab" not in sys.modules: diff --git a/bench/gen_pdf.py b/bench/gen_pdf.py index 903eb92..772424e 100644 --- a/bench/gen_pdf.py +++ b/bench/gen_pdf.py @@ -23,7 +23,6 @@ from reportlab.platypus import ( SimpleDocTemplate, Table, TableStyle, Paragraph, Spacer, ) -from reportlab.lib.enums import TA_CENTER OUTPUT = os.path.join(os.path.dirname(__file__), "..", "bench_results.pdf") @@ -123,7 +122,6 @@ def assign_sections(names: list[str]) -> dict[str, str]: GREEN = colors.HexColor("#2e7d32") RED = colors.HexColor("#c62828") -LIGHT_GREEN = colors.HexColor("#e8f5e9") LIGHT_RED = colors.HexColor("#ffebee") HEADER_BG = colors.HexColor("#263238") SECTION_BG = colors.HexColor("#eceff1") diff --git a/pixi.toml b/pixi.toml index 82612c9..e0d3ebc 100644 --- a/pixi.toml +++ b/pixi.toml @@ -42,7 +42,6 @@ MBLACK_CACHE_DIR = "$CONDA_PREFIX/.mblack-cache" [tasks] test = { cmd = "python3 run_test.py" } -build = { cmd = "mojo precompile emberregex -o emberregex.mojoc"} bench = { cmd = "mojo -I . bench/bench.mojo" } bench_set = { cmd = "mojo -I . bench/bench_set.mojo" } bench_all = {depends-on=["bench", "bench_set"]} diff --git a/tools/comptime_stages.py b/tools/comptime_stages.py index 10c05db..3d9610f 100644 --- a/tools/comptime_stages.py +++ b/tools/comptime_stages.py @@ -20,7 +20,7 @@ Usage: python3 tools/comptime_stages.py # default pattern sets python3 tools/comptime_stages.py rose_log # one set - python3 tools/comptime_stages.py --repeat 3 # median of 3 + python3 tools/comptime_stages.py --repeat 3 # min of 3 Read the DELTA column, not the absolute times: every row pays the same fixed Mojo startup + codegen baseline. @@ -32,7 +32,6 @@ """ import argparse -import statistics import sys import tempfile From 89a852ab1af79ef49cfb4810333c926a8cc3e5fe Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:10:10 -0600 Subject: [PATCH 31/65] cleanup: drop dead imports and stale comments in engine.mojo Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/engine.mojo | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/emberregex/engine.mojo b/emberregex/engine.mojo index fbbf2ad..0716d94 100644 --- a/emberregex/engine.mojo +++ b/emberregex/engine.mojo @@ -23,11 +23,9 @@ from .constants import ( CHAR_STAR, CHAR_ZERO, ) -from .parser import parse from .nfa import ( _build_static_nfa, _nfa_has_backref, - build_nfa, split_cycle_flags, NFA, NFAStateKind, @@ -49,9 +47,6 @@ from .optimize import ( lit_bytes_arr, lit_flags_arr, select_probe_offsets, - FilterPrefix, - InnerLiteral, - LiteralAlt, ) from .teddy import ( teddy_find_prefix, @@ -60,7 +55,6 @@ from .teddy import ( teddy_search_forward, ) from .simd_scan import ( - clear_first_lane, first_lane_index, lane_bits, simd_find_byte, @@ -125,7 +119,6 @@ from .simd_kernels import ( ) from .executor import PikeVM, _VMBuffers, heapbt_match from .onepass import ( - OnePass, build_onepass, onepass_shape, onepass_class_arr, @@ -672,10 +665,6 @@ def _lf_end_deterministic_list(nfa: NFA) -> Bool: return True -# Sometimes this produces better IR since the __init__ gets folded into -# a constant. - - @always_inline def _bm_word(bm: SIMD[DType.uint8, BITMAP_WIDTH], k: Int) -> UInt64: """Bytes [8k, 8k+8) of a charset bitmap as one little-endian word.""" @@ -862,6 +851,8 @@ def _lf_end_deterministic[fast: Bool = True](nfa: NFA) -> Bool: return _lf_end_deterministic_list(nfa) +# Fresh capture slots: `materialize[ALL_NEG_ONES[n]]()` sometimes gives +# better IR than `Array(fill=-1)`, the initializer folding to a constant. comptime ALL_NEG_ONES[Size: Int] = Array[Int, Size](fill=-1) # Steps the leftmost-first lane's speculative backtracker attempt may @@ -912,12 +903,6 @@ def __literal_can_be_optimized(width: Int) -> Bool: comptime TypeForPrefixLength[width: Int] = SIMD[Byte.dtype, width] -# The probe compares (_probe_eq/_probe_eq1) live in simd_scan.mojo as -# probe_eq/probe_eq1, next to simd_find_literal_rare — the lifted Mula -# memmem both the filter-prefix scanner and the inner-literal strategy -# call. - - def _dfa_candidate(nfa: NFA, cyclic: List[Bool]) -> Bool: """True when the pattern's SHAPE should run on a DFA engine (eager or lazy): capture-free, this is the classic/leftmost-first lanes @@ -1596,7 +1581,6 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( comptime if Self._use_lazy_dfa: var nfa = materialize[Self.nfa]() self._dfa_nfa = rebind_var[type_of(self._dfa_nfa)](nfa^) - # self._dfa_nfa = rebind_var[type_of(self._dfa_nfa)](materialize[_build_static_nfa(Self.pattern)]()) var dfa = LazyDFA() self._dfa = rebind_var[type_of(self._dfa)](dfa^) else: From 6d9482fdb64097e5ae57596b77e7ce1a7b09317d Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:10:39 -0600 Subject: [PATCH 32/65] cleanup: CI runs the tooling unit tests tools/test_run_test.py and tools/test_run_coverage.py passed but nothing ran them. One stdlib-only step on the Linux leg of the Tests workflow, before pixi is installed, so a runner regression fails in seconds. Validated with actionlint. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a8ab694..fdf16bb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,6 +36,12 @@ jobs: - uses: actions/checkout@v4 + # Unit tests for run_test.py / run_coverage.py: stdlib-only Python, + # so the runner's own interpreter will do, and one OS is enough. + - name: Test the test tooling + if: runner.os == 'Linux' + run: python3 -m unittest discover -s tools -p 'test_*.py' + - name: Install pixi id: pixi run: | From 175887b89d89195bcac7210406bffe6116949ffb Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:10:41 -0600 Subject: [PATCH 33/65] cleanup: build the iota/bit/salt lane constants with std.math.iota Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/static_dfa.mojo | 58 +++++++----------------------------- emberregex/static_lfdfa.mojo | 10 ++----- 2 files changed, 13 insertions(+), 55 deletions(-) diff --git a/emberregex/static_dfa.mojo b/emberregex/static_dfa.mojo index c4299bb..1363815 100644 --- a/emberregex/static_dfa.mojo +++ b/emberregex/static_dfa.mojo @@ -13,6 +13,7 @@ which keeps its own 4096-state cap and Pike VM fallback. from std.bit import count_trailing_zeros, pop_count from std.collections import Array +from std.math import iota from std.sys import simd_width_of from .ast import AnchorKind @@ -212,16 +213,9 @@ def _bs_eq(a: _StateBits, b: _StateBits) -> Bool: return (a ^ b).reduce_or() == 0 -def _mk_bs_salt() -> SIMD[DType.uint64, 64]: - """Distinct odd multiplier per lane so identical words in different - lanes hash differently.""" - var v = SIMD[DType.uint64, 64](0) - for i in range(64): - v[i] = UInt64(2 * i + 1) * 0x9E3779B97F4A7C15 - return v - - -comptime _BS_SALT = _mk_bs_salt() +# Distinct odd multiplier per lane so identical words in different lanes +# hash differently. +comptime _BS_SALT = (2 * iota[DType.uint64, 64]() + 1) * 0x9E3779B97F4A7C15 @always_inline @@ -748,14 +742,7 @@ def _classic_flags( return fl -def _mk_iota256_i64() -> SIMD[DType.int64, 256]: - var v = SIMD[DType.int64, 256](0) - for i in range(256): - v[i] = Int64(i) - return v - - -comptime _IOTA256_I64 = _mk_iota256_i64() +comptime _IOTA256_I64 = iota[DType.int64, 256]() def _byte_classes(nfa: NFA, mut class_of: List[Int]) -> List[Int]: @@ -939,35 +926,12 @@ comptime _MIN_CAP = 128 comptime _COL_CHUNK = 8 -def _mk_bit64() -> SIMD[DType.uint64, 64]: - var v = SIMD[DType.uint64, 64](0) - for i in range(64): - v[i] = UInt64(1) << UInt64(i) - return v - - -comptime _BIT64 = _mk_bit64() - - -def _mk_iota256() -> SIMD[DType.int32, 256]: - var v = SIMD[DType.int32, 256](0) - for i in range(256): - v[i] = Int32(i) - return v - - -comptime _IOTA256 = _mk_iota256() - - -def _mk_col_salt() -> SIMD[DType.uint64, _MIN_CAP]: - """Distinct odd multiplier per state lane, as in _mk_bs_salt.""" - var v = SIMD[DType.uint64, _MIN_CAP](0) - for i in range(_MIN_CAP): - v[i] = UInt64(2 * i + 1) * 0x9E3779B97F4A7C15 - return v - - -comptime _COL_SALT = _mk_col_salt() +comptime _BIT64 = SIMD[DType.uint64, 64](1) << iota[DType.uint64, 64]() +comptime _IOTA256 = iota[DType.int32, 256]() +# Distinct odd multiplier per state lane, as in _BS_SALT. +comptime _COL_SALT = ( + 2 * iota[DType.uint64, _MIN_CAP]() + 1 +) * 0x9E3779B97F4A7C15 @always_inline diff --git a/emberregex/static_lfdfa.mojo b/emberregex/static_lfdfa.mojo index 9f69bd2..73b7fdd 100644 --- a/emberregex/static_lfdfa.mojo +++ b/emberregex/static_lfdfa.mojo @@ -46,6 +46,7 @@ Continuation closures are memoized per target state in one flat pool. from std.bit import count_leading_zeros, count_trailing_zeros from std.collections import Array +from std.math import iota from .ast import AnchorKind from .constants import CHAR_NEWLINE @@ -215,14 +216,7 @@ comptime _LF_TAIL_SALT_O: UInt64 = 0x165667B19E3779F9 comptime _LF_TAIL_SALT_N: UInt64 = 0x27D4EB2F165667C5 -def _mk_iota64() -> SIMD[DType.uint64, 64]: - var v = SIMD[DType.uint64, 64](0) - for i in range(64): - v[i] = UInt64(i) - return v - - -comptime _IOTA64 = _mk_iota64() +comptime _IOTA64 = iota[DType.uint64, 64]() def _lf_memo_closure( From 3f5a43cacf122827348b79dc5fd0ad7b4a6d1586 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:11:52 -0600 Subject: [PATCH 34/65] cleanup: one AccelSet for the eager, reverse, set and one-pass acceleration data The 7 accel_* fields, their mask-word and has-accel helpers and the count-exits / collect-exits / <=2-compares-or-shufti/truffle block were copied across EagerDFA, RDFA, MultiDFA and the one-pass builder; the region's exit encoding and build_class_masks repeated the shufti-or-truffle branch. Now AccelSet (add / mask_word / any), accel_exits and build_nib_masks in simd_kernels. Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 2 +- emberregex/onepass.mojo | 39 +----- emberregex/set_dfa.mojo | 96 ++++----------- emberregex/simd_kernels.mojo | 104 +++++++++++++--- emberregex/static_dfa.mojo | 178 ++++++++-------------------- emberregex/static_rdfa.mojo | 95 ++++----------- test/test_bench_coverage.mojo | 2 +- test/test_dfa_builders_runtime.mojo | 44 +++---- test/test_leftmost_first_dfa.mojo | 14 +-- test/test_onepass.mojo | 2 +- test/test_set_build_parity.mojo | 14 +-- test/test_set_phase2.mojo | 4 +- test/test_simd_kernels.mojo | 12 +- 13 files changed, 230 insertions(+), 376 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 441e642..335a642 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -119,7 +119,7 @@ current per-file durations. test can drift onto a different engine and keep passing — it just quietly stops testing what its filename claims. Pin with `_strategy.use_*`, `_use_lf_dfa`, `_use_onepass`, `_use_dfa_span`, `_SHENG_CAP`, - `accel_nib_states`, or the engine's own runtime state (`clear_count`). + `accel.nib_states`, or the engine's own runtime state (`clear_count`). - **Pin exclusively, never with a disjunction.** `assert_true(use_teddy or use_eager_dfa)` passes on either arm and therefore cannot catch a lane change. Use `comptime if HAS_FAST_BYTE_SHUFFLE:` with the diff --git a/emberregex/onepass.mojo b/emberregex/onepass.mojo index 837b0d8..1c7e236 100644 --- a/emberregex/onepass.mojo +++ b/emberregex/onepass.mojo @@ -102,14 +102,7 @@ from .static_dfa import ( _nfa_has_word_anchor, _wb_holds, ) -from .simd_kernels import ( - ACCEL_SHUFTI, - ACCEL_TRUFFLE, - HAS_FAST_BYTE_SHUFFLE, - build_shufti_masks, - build_truffle_masks, - shufti_encodable, -) +from .simd_kernels import accel_exits # Caps. A DFA state is one NFA transition target in one context, so the # state count is bounded by the NFA; the cap keeps the table (and the @@ -170,7 +163,7 @@ struct OnePass(Copyable, Movable): var start_other: Int var start_word: Int var any_need: Bool # some match state carries an OP_NEED_* bit - # Acceleration view: only `accel_*`, `num_states` and + # Acceleration view: only `accel`, `num_states` and # `num_match_states` (0: no match bookkeeping) are filled in, for # `_edfa_accel_skip`. var accel: EagerDFA @@ -613,31 +606,9 @@ def build_onepass(nfa: NFA, enabled: Bool) -> OnePass: var cell = s * nclasses + c if trans_next[cell] == s and trans_eps[cell] == 0: loopb |= classv.eq(SIMD[DType.int64, 256](c)) - var exits = List[Int]() - var loops = 0 - for b in range(256): - if loopb[b]: - loops += 1 - else: - exits.append(b) - if loops == 0 or len(exits) == 0: - continue - if len(exits) <= 2: - accel.accel_states.append(s) - accel.accel_exit1.append(exits[0]) - accel.accel_exit2.append(exits[1] if len(exits) == 2 else -1) - elif HAS_FAST_BYTE_SHUFFLE: - var t0 = List[Int]() - var t1 = List[Int]() - if shufti_encodable(exits): - build_shufti_masks(exits, t0, t1) - accel.accel_nib_kind.append(ACCEL_SHUFTI) - else: - build_truffle_masks(exits, t0, t1) - accel.accel_nib_kind.append(ACCEL_TRUFFLE) - accel.accel_nib_states.append(s) - accel.accel_nib_t0.extend(t0^) - accel.accel_nib_t1.extend(t1^) + var exits = accel_exits(~loopb) + if len(exits) > 0: + accel.accel.add(s, exits) var any_need = False for s in range(num_states): diff --git a/emberregex/set_dfa.mojo b/emberregex/set_dfa.mojo index 5e7cd6e..7017d33 100644 --- a/emberregex/set_dfa.mojo +++ b/emberregex/set_dfa.mojo @@ -38,15 +38,11 @@ from .dfa import _epsilon_closure from .nfa import NFA, NFAStateKind from .set_pike import SetMatch from .simd_kernels import ( - ACCEL_SHUFTI, - ACCEL_TRUFFLE, - HAS_FAST_BYTE_SHUFFLE, + AccelSet, _class_contains, - build_shufti_masks, - build_truffle_masks, + accel_exits, find_in_class, nibble_table_from, - shufti_encodable, ) from .simd_scan import first_lane_index, lane_bits from .static_bytes import table_bytes @@ -89,15 +85,7 @@ struct MultiDFA(Copyable, Movable): var end_len: List[Int] var any_nl: Bool var any_end: Bool - # Accelerated states: self-loop on all but <= 2 bytes. - var accel_states: List[Int] - var accel_exit1: List[Int] - var accel_exit2: List[Int] - # Nibble-accelerated states (arbitrary exit sets, shuffle targets). - var accel_nib_states: List[Int] - var accel_nib_kind: List[Int] - var accel_nib_t0: List[Int] - var accel_nib_t1: List[Int] + var accel: AccelSet def __init__(out self): """Invalid placeholder with one state (keeps downstream @@ -116,13 +104,7 @@ struct MultiDFA(Copyable, Movable): self.end_len = List[Int](fill=0, length=1) self.any_nl = False self.any_end = False - self.accel_states = List[Int]() - self.accel_exit1 = List[Int]() - self.accel_exit2 = List[Int]() - self.accel_nib_states = List[Int]() - self.accel_nib_kind = List[Int]() - self.accel_nib_t0 = List[Int]() - self.accel_nib_t1 = List[Int]() + self.accel = AccelSet() def _sorted_dedup(var ids: List[Int]) -> List[Int]: @@ -582,33 +564,11 @@ def _mdfa_finish( for s in range(n): if s < num_report: continue - var row = new_rows.unsafe_get(s) - var exit_count = 0 - for byte in range(256): - if Int(row[byte]) != s: - exit_count += 1 - if exit_count == 0 or exit_count == 256: - continue - var exits = List[Int]() - for byte in range(256): - if Int(row[byte]) != s: - exits.append(byte) - if len(exits) <= 2: - result.accel_states.append(s) - result.accel_exit1.append(exits[0]) - result.accel_exit2.append(exits[1] if len(exits) == 2 else -1) - elif HAS_FAST_BYTE_SHUFFLE: - var t0 = List[Int]() - var t1 = List[Int]() - if shufti_encodable(exits): - build_shufti_masks(exits, t0, t1) - result.accel_nib_kind.append(ACCEL_SHUFTI) - else: - build_truffle_masks(exits, t0, t1) - result.accel_nib_kind.append(ACCEL_TRUFFLE) - result.accel_nib_states.append(s) - result.accel_nib_t0.extend(t0^) - result.accel_nib_t1.extend(t1^) + var exits = accel_exits( + new_rows.unsafe_get(s).ne(SIMD[DType.int32, 256](Int32(s))) + ) + if len(exits) > 0: + result.accel.add(s, exits) # One 256-lane vector store per row (see static_dfa `_edfa_finish`). var new_table = List[Int](fill=-1, length=n * 256) @@ -742,18 +702,6 @@ def mdfa_slices_arr[n: Int](d: MultiDFA) -> Array[Int32, n]: # --- Runtime walker ---------------------------------------------------------- -def _maccel_mask_word(d: MultiDFA, word: Int) -> UInt64: - """Comptime: bitmask of accelerated state ids in [word*64, ...).""" - var m = UInt64(0) - for s in d.accel_states: - if s >> 6 == word: - m |= UInt64(1) << UInt64(s & 63) - for s in d.accel_nib_states: - if s >> 6 == word: - m |= UInt64(1) << UInt64(s & 63) - return m - - @always_inline def _mdfa_accel_skip[ origin: Origin, //, d: MultiDFA @@ -769,7 +717,7 @@ def _mdfa_accel_skip[ comptime NW = (d.num_states + 63) >> 6 var in_accel = False comptime for w in range(NW): - comptime mw = _maccel_mask_word(d, w) + comptime mw = d.accel.mask_word(w) comptime if mw != 0: if (cur >> 6) == w: in_accel = ((mw >> UInt64(cur & 63)) & 1) != 0 @@ -777,19 +725,19 @@ def _mdfa_accel_skip[ return pos var p = pos - comptime for ai in range(len(d.accel_states)): - comptime a_state = d.accel_states[ai] - comptime a_e1 = UInt8(d.accel_exit1[ai]) + comptime for ai in range(len(d.accel.states)): + comptime a_state = d.accel.states[ai] + comptime a_e1 = UInt8(d.accel.exit1[ai]) comptime a_e2 = UInt8( - d.accel_exit2[ai] if d.accel_exit2[ai] >= 0 else d.accel_exit1[ai] + d.accel.exit2[ai] if d.accel.exit2[ai] >= 0 else d.accel.exit1[ai] ) if cur == a_state: p = _find_exit2[e1=a_e1, e2=a_e2](input, p) - comptime for ai in range(len(d.accel_nib_states)): - comptime a_state = d.accel_nib_states[ai] - comptime a_kind = d.accel_nib_kind[ai] - comptime a_t0 = nibble_table_from(d.accel_nib_t0, ai) - comptime a_t1 = nibble_table_from(d.accel_nib_t1, ai) + comptime for ai in range(len(d.accel.nib_states)): + comptime a_state = d.accel.nib_states[ai] + comptime a_kind = d.accel.nib_kind[ai] + comptime a_t0 = nibble_table_from(d.accel.nib_t0, ai) + comptime a_t1 = nibble_table_from(d.accel.nib_t1, ai) if cur == a_state: # Scalar peek: only vectorize when the current byte actually # self-loops; instant exits go back to the table walk. @@ -800,10 +748,6 @@ def _mdfa_accel_skip[ return p -def _mdfa_has_accel(d: MultiDFA) -> Bool: - return len(d.accel_states) > 0 or len(d.accel_nib_states) > 0 - - @always_inline def _emit_merged[ pn: Int, //, pool: Array[Int32, pn] @@ -915,7 +859,7 @@ def mdfa_scan[ is emitted; the callers (`_scan_ladder`, the bench's phase-2 helper) each have exactly one call.""" var out = List[SetMatch]() - comptime if _mdfa_has_accel(d): + comptime if d.accel.any(): comptime W = simd_width_of[DType.uint8]() if len(input) >= W: _mdfa_scan_impl[ diff --git a/emberregex/simd_kernels.mojo b/emberregex/simd_kernels.mojo index 08227a8..7ff6fd9 100644 --- a/emberregex/simd_kernels.mojo +++ b/emberregex/simd_kernels.mojo @@ -212,26 +212,100 @@ def stops_from_bitmap(bitmap: SIMD[DType.uint8, BITMAP_WIDTH]) -> List[Int]: return stops^ +def build_nib_masks( + stop_bytes: List[Int], mut t0: List[Int], mut t1: List[Int] +) -> Int: + """Comptime: encode a byte set as nibble masks — shufti when exact, + truffle otherwise — and return the kind (ACCEL_SHUFTI/_TRUFFLE).""" + if shufti_encodable(stop_bytes): + build_shufti_masks(stop_bytes, t0, t1) + return ACCEL_SHUFTI + build_truffle_masks(stop_bytes, t0, t1) + return ACCEL_TRUFFLE + + def build_class_masks( stop_bytes: List[Int], ) -> Tuple[Int, _NibbleTable, _NibbleTable]: - """Comptime: encode a byte set as (kind, t0, t1) for find_in_class — - shufti when exact, truffle otherwise.""" + """Comptime: encode a byte set as (kind, t0, t1) for find_in_class.""" var t0 = List[Int]() var t1 = List[Int]() - if shufti_encodable(stop_bytes): - build_shufti_masks(stop_bytes, t0, t1) - return ( - ACCEL_SHUFTI, - nibble_table_from(t0, 0), - nibble_table_from(t1, 0), - ) - build_truffle_masks(stop_bytes, t0, t1) - return ( - ACCEL_TRUFFLE, - nibble_table_from(t0, 0), - nibble_table_from(t1, 0), - ) + var kind = build_nib_masks(stop_bytes, t0, t1) + return (kind, nibble_table_from(t0, 0), nibble_table_from(t1, 0)) + + +def accel_exits(exit_lanes: SIMD[DType.bool, 256]) -> List[Int]: + """Comptime: the exit bytes (set lanes) of a state that self-loops on + the other bytes, or empty when it never exits or never self-loops — + nothing to skip either way.""" + var n = 0 + for b in range(256): + if exit_lanes[b]: + n += 1 + var exits = List[Int]() + if n == 0 or n == 256: + return exits^ + for b in range(256): + if exit_lanes[b]: + exits.append(b) + return exits^ + + +struct AccelSet(Copyable, Movable): + """Comptime acceleration data of a table: states that self-loop on all + but an exit-byte set, which the walkers SIMD-scan to the next exit + byte instead of stepping the table. <= 2 exit bytes (e.g. the `.*` + state of `.*x`) use direct compares; larger sets (e.g. the `\\w+` + self-loop) are nibble-encoded (`build_nib_masks`), only on targets + with a native byte shuffle.""" + + var states: List[Int] + var exit1: List[Int] # first exit byte per state + var exit2: List[Int] # second exit byte, or -1 if only one + var nib_states: List[Int] + var nib_kind: List[Int] # ACCEL_SHUFTI or ACCEL_TRUFFLE + var nib_t0: List[Int] # NIBBLE_TABLE_SIZE entries per state + var nib_t1: List[Int] # NIBBLE_TABLE_SIZE entries per state + + def __init__(out self): + self.states = List[Int]() + self.exit1 = List[Int]() + self.exit2 = List[Int]() + self.nib_states = List[Int]() + self.nib_kind = List[Int]() + self.nib_t0 = List[Int]() + self.nib_t1 = List[Int]() + + def add(mut self, s: Int, exits: List[Int]): + """Comptime: accelerate state `s` over its non-empty exit set + (`accel_exits`); a no-op for > 2 exits off shuffle targets.""" + if len(exits) <= 2: + self.states.append(s) + self.exit1.append(exits[0]) + self.exit2.append(exits[1] if len(exits) == 2 else -1) + elif HAS_FAST_BYTE_SHUFFLE: + var t0 = List[Int]() + var t1 = List[Int]() + self.nib_kind.append(build_nib_masks(exits, t0, t1)) + self.nib_states.append(s) + self.nib_t0.extend(t0^) + self.nib_t1.extend(t1^) + + def mask_word(self, word: Int) -> UInt64: + """Comptime: bitmask of accelerated state ids in + [word*64, (word+1)*64).""" + var m = UInt64(0) + for s in self.states: + if s >> 6 == word: + m |= UInt64(1) << UInt64(s & 63) + for s in self.nib_states: + if s >> 6 == word: + m |= UInt64(1) << UInt64(s & 63) + return m + + def any(self) -> Bool: + """Comptime: does any state carry acceleration data?""" + return len(self.states) > 0 or len(self.nib_states) > 0 # --- Scanners --------------------------------------------------------------- diff --git a/emberregex/static_dfa.mojo b/emberregex/static_dfa.mojo index 43311ef..dc3c7a0 100644 --- a/emberregex/static_dfa.mojo +++ b/emberregex/static_dfa.mojo @@ -24,19 +24,13 @@ from .dfa import _reaches_match from .charset import BITMAP_WIDTH from .simd_scan import first_lane_index, lane_bits, simd_find_byte from .simd_kernels import ( - ACCEL_SHUFTI, - ACCEL_TRUFFLE, - HAS_FAST_BYTE_SHUFFLE, + AccelSet, _ShuffleIndex, _class_contains, _sheng_step, - build_class_masks, - build_shufti_masks, - build_truffle_masks, + accel_exits, find_in_class, nibble_table_from, - shufti_encodable, - stops_from_bitmap, ) # Per-state flag bits (see EagerDFA.flags) @@ -611,18 +605,8 @@ struct EagerDFA(Copyable, Movable): var any_eol_nl: Bool # some state carries EDFA_EOL_AT_NEWLINE var any_eol_end: Bool # some state carries EDFA_EOL_AT_END var any_wb: Bool # some state carries EDFA_MATCH_IF_WORD / _NONWORD - # Accelerated states: self-loop on all but <= 2 bytes. The walkers - # SIMD-scan to the next exit byte instead of stepping the table. - var accel_states: List[Int] - var accel_exit1: List[Int] # first exit byte per accelerated state - var accel_exit2: List[Int] # second exit byte, or -1 if only one - # Nibble-accelerated states: self-loop on all but an arbitrary exit-byte - # set, encoded as shufti or truffle masks (see simd_kernels.mojo). Only - # populated when the target has a native byte shuffle. - var accel_nib_states: List[Int] - var accel_nib_kind: List[Int] # ACCEL_SHUFTI or ACCEL_TRUFFLE - var accel_nib_t0: List[Int] # NIBBLE_TABLE_SIZE entries per state - var accel_nib_t1: List[Int] # NIBBLE_TABLE_SIZE entries per state + # Accelerated states (see AccelSet). + var accel: AccelSet # Region acceleration: a small set of flag-free states whose rows # agree on every byte outside an exit set and land inside the set — # the look-behind-split restart states of a word-anchor pattern @@ -631,11 +615,7 @@ struct EagerDFA(Copyable, Movable): # The walkers SIMD-scan to the next exit byte as for a single state # and land in the member the last skipped byte selects. var region_states: List[Int] - var region_exit1: Int # -1 when the exit set is nibble-encoded - var region_exit2: Int # or -1 - var region_nib_kind: Int - var region_nib_t0: List[Int] - var region_nib_t1: List[Int] + var region_exits: AccelSet # the region's exit set, as one entry var region_land: List[Int] # 256 entries: member landed in per byte def __init__(out self): @@ -654,19 +634,9 @@ struct EagerDFA(Copyable, Movable): self.any_eol_nl = False self.any_eol_end = False self.any_wb = False - self.accel_states = List[Int]() - self.accel_exit1 = List[Int]() - self.accel_exit2 = List[Int]() - self.accel_nib_states = List[Int]() - self.accel_nib_kind = List[Int]() - self.accel_nib_t0 = List[Int]() - self.accel_nib_t1 = List[Int]() + self.accel = AccelSet() self.region_states = List[Int]() - self.region_exit1 = -1 - self.region_exit2 = -1 - self.region_nib_kind = 0 - self.region_nib_t0 = List[Int]() - self.region_nib_t1 = List[Int]() + self.region_exits = AccelSet() self.region_land = List[Int]() @@ -1803,31 +1773,16 @@ def _edfa_finish( # bytes 80 -> 22 us, 4 exits 80 -> 45 us, 8 exits 79 -> 89 us # (slower), 26 exits (`\b[a-z]+ing\b`) 101 -> 128 us (slower). if len(exits) > 0 and len(exits) <= _REGION_MAX_EXITS: - var encodable = len(exits) <= 2 or HAS_FAST_BYTE_SHUFFLE - if encodable: + var region_exits = AccelSet() + region_exits.add(members[0], exits) + if region_exits.any(): # encodable on this target result.region_states = members.copy() result.region_land = land^ - if len(exits) <= 2: - result.region_exit1 = exits[0] - result.region_exit2 = exits[1] if len(exits) == 2 else -1 - else: - var t0 = List[Int]() - var t1 = List[Int]() - if shufti_encodable(exits): - build_shufti_masks(exits, t0, t1) - result.region_nib_kind = ACCEL_SHUFTI - else: - build_truffle_masks(exits, t0, t1) - result.region_nib_kind = ACCEL_TRUFFLE - result.region_nib_t0 = t0^ - result.region_nib_t1 = t1^ - - # Acceleration: a state that self-loops on all but an exit-byte set gets - # a SIMD scan to its next exit byte instead of a per-byte table walk. - # <= 2 exit bytes (e.g. the `.*` state of `.*x`) use direct compares; - # larger sets (e.g. the `\w+` self-loop) are nibble-encoded as shufti - # masks when exact, truffle otherwise — only on targets with a native - # byte shuffle. EOL_AT_NEWLINE-flagged states are excluded: skipping + result.region_exits = region_exits^ + + # Acceleration (see AccelSet): a state that self-loops on all but an + # exit-byte set gets a SIMD scan to its next exit byte instead of a + # per-byte table walk. EOL_AT_NEWLINE-flagged states are excluded: skipping # bytes would skip their per-'\n' last_match updates when '\n' # self-loops. So are states the producer vetoed (EDFA_NO_ACCEL) and # states whose match-ness depends on the next byte's word class @@ -1844,13 +1799,6 @@ def _edfa_finish( != 0 ): continue - var row = new_rows.unsafe_get(s) - var exit_count = 0 - for byte in range(256): - if Int(row[byte]) != s: - exit_count += 1 - if exit_count == 0 or exit_count == 256: - continue # never exits / never self-loops: nothing to skip # A region member takes the region skip instead: its own loop # set is a byte class (word bytes, say) whose runs are a few # bytes long in prose, while the region's exit set is sparse — @@ -1862,26 +1810,11 @@ def _edfa_finish( is_member = True if is_member: continue - var exits = List[Int]() - for byte in range(256): - if Int(row[byte]) != s: - exits.append(byte) - if len(exits) <= 2: - result.accel_states.append(s) - result.accel_exit1.append(exits[0]) - result.accel_exit2.append(exits[1] if len(exits) == 2 else -1) - elif HAS_FAST_BYTE_SHUFFLE: - var t0 = List[Int]() - var t1 = List[Int]() - if shufti_encodable(exits): - build_shufti_masks(exits, t0, t1) - result.accel_nib_kind.append(ACCEL_SHUFTI) - else: - build_truffle_masks(exits, t0, t1) - result.accel_nib_kind.append(ACCEL_TRUFFLE) - result.accel_nib_states.append(s) - result.accel_nib_t0.extend(t0^) - result.accel_nib_t1.extend(t1^) + var exits = accel_exits( + new_rows.unsafe_get(s).ne(SIMD[DType.int32, 256](Int32(s))) + ) + if len(exits) > 0: + result.accel.add(s, exits) # The veto has done its job; the walkers' flag bytes never carry it. for s in range(nsets): new_flags[s] &= ~Int(EDFA_NO_ACCEL) @@ -1971,18 +1904,6 @@ def _find_exit2[ return input_len -def _accel_mask_word(d: EagerDFA, word: Int) -> UInt64: - """Comptime: bitmask of accelerated state ids in [word*64, (word+1)*64).""" - var m = UInt64(0) - for s in d.accel_states: - if s >> 6 == word: - m |= UInt64(1) << UInt64(s & 63) - for s in d.accel_nib_states: - if s >> 6 == word: - m |= UInt64(1) << UInt64(s & 63) - return m - - def _region_land_arr(d: EagerDFA) -> Array[Int16, 256]: """Comptime: `region_land` as a materializable array.""" var arr = Array[Int16, 256](fill=-1) @@ -2011,8 +1932,8 @@ def _edfa_accel_skip[ comptime W = simd_width_of[DType.uint8]() if pos + W > len(input): return pos - comptime m0 = _accel_mask_word(d, 0) - comptime m1 = _accel_mask_word(d, 1) + comptime m0 = d.accel.mask_word(0) + comptime m1 = d.accel.mask_word(1) comptime if d.num_states <= 64: if (m0 >> UInt64(cur)) & 1 == 0: return pos @@ -2022,21 +1943,21 @@ def _edfa_accel_skip[ return pos var p = pos - comptime for ai in range(len(d.accel_states)): - comptime a_state = d.accel_states[ai] - comptime a_e1 = UInt8(d.accel_exit1[ai]) + comptime for ai in range(len(d.accel.states)): + comptime a_state = d.accel.states[ai] + comptime a_e1 = UInt8(d.accel.exit1[ai]) comptime a_e2 = UInt8( - d.accel_exit2[ai] if d.accel_exit2[ai] >= 0 else d.accel_exit1[ai] + d.accel.exit2[ai] if d.accel.exit2[ai] >= 0 else d.accel.exit1[ai] ) if cur == a_state: p = _find_exit2[e1=a_e1, e2=a_e2](input, p) comptime if a_state < d.num_match_states: last_match = p - comptime for ai in range(len(d.accel_nib_states)): - comptime a_state = d.accel_nib_states[ai] - comptime a_kind = d.accel_nib_kind[ai] - comptime a_t0 = nibble_table_from(d.accel_nib_t0, ai) - comptime a_t1 = nibble_table_from(d.accel_nib_t1, ai) + comptime for ai in range(len(d.accel.nib_states)): + comptime a_state = d.accel.nib_states[ai] + comptime a_kind = d.accel.nib_kind[ai] + comptime a_t0 = nibble_table_from(d.accel.nib_t0, ai) + comptime a_t1 = nibble_table_from(d.accel.nib_t1, ai) if cur == a_state: # Scalar peek: only vectorize when the current byte actually # self-loops; instant exits go back to the table walk. @@ -2072,10 +1993,11 @@ def _edfa_region_skip[ in_region = True if in_region and p < len(input): var p2 = p - comptime if d.region_exit1 >= 0: - comptime r_e1 = UInt8(d.region_exit1) + comptime if len(d.region_exits.states) > 0: + comptime r_e1 = UInt8(d.region_exits.exit1[0]) comptime r_e2 = UInt8( - d.region_exit2 if d.region_exit2 >= 0 else d.region_exit1 + d.region_exits.exit2[0] if d.region_exits.exit2[0] + >= 0 else d.region_exits.exit1[0] ) # Scalar peek: a region is re-entered right after landing # on an exit byte (a false candidate), where a vector @@ -2084,9 +2006,9 @@ def _edfa_region_skip[ if b0 != r_e1 and b0 != r_e2: p2 = _find_exit2[e1=r_e1, e2=r_e2](input, p + 1) else: - comptime r_kind = d.region_nib_kind - comptime r_t0 = nibble_table_from(d.region_nib_t0, 0) - comptime r_t1 = nibble_table_from(d.region_nib_t1, 0) + comptime r_kind = d.region_exits.nib_kind[0] + comptime r_t0 = nibble_table_from(d.region_exits.nib_t0, 0) + comptime r_t1 = nibble_table_from(d.region_exits.nib_t1, 0) if not _class_contains[kind=r_kind, t0=r_t0, t1=r_t1]( input.unsafe_get(p) ): @@ -2110,15 +2032,11 @@ def _edfa_has_region(d: EagerDFA) -> Bool: def _edfa_has_accel(d: EagerDFA) -> Bool: """Comptime: does any state carry acceleration data?""" - return ( - len(d.accel_states) > 0 - or len(d.accel_nib_states) > 0 - or len(d.region_states) >= 2 - ) + return d.accel.any() or len(d.region_states) >= 2 def _start_run_skip_idx(d: EagerDFA) -> Int: - """Comptime: index into accel_nib_* of a self-looping nib-accel state + """Comptime: index into accel.nib_* of a self-looping nib-accel state `S1` such that the mid-line start state (`start_other`, `S0`) transitions to `S1` on *every* byte that `S1` self-loops on — else -1. @@ -2145,8 +2063,8 @@ def _start_run_skip_idx(d: EagerDFA) -> Int: .unsafe_bitcast[Int64]() .unsafe_load[width=256]() ) - for i in range(len(d.accel_nib_states)): - var s1 = d.accel_nib_states[i] + for i in range(len(d.accel.nib_states)): + var s1 = d.accel.nib_states[i] var row1 = ( Pointer(to=d.table[s1 * 256]) .unsafe_bitcast[Int64]() @@ -2172,7 +2090,7 @@ def _start_run_skip_idx(d: EagerDFA) -> Int: def _pivot_prefilter(d: EagerDFA) -> Tuple[Int, Int]: - """Comptime: (accel_nib index of S1, pivot byte P) enabling the + """Comptime: (accel.nib_* index of S1, pivot byte P) enabling the pivot-anchored search prefilter, or (-1, -1). Qualifying shape — the `[class]+ P …` family (e.g. an email regex's @@ -2193,7 +2111,7 @@ def _pivot_prefilter(d: EagerDFA) -> Tuple[Int, Int]: var rs = _start_run_skip_idx(d) if rs < 0: return (-1, -1) - var s1 = d.accel_nib_states[rs] + var s1 = d.accel.nib_states[rs] var n = d.num_states # Rows as 256-lane vectors throughout: the cell-by-cell form read the # whole table several times over at ~50 us per List element. @@ -2294,7 +2212,7 @@ def _pivot_forced_chain(d: EagerDFA, pv: Tuple[Int, Int]) -> List[Int]: var chain = List[Int]() if pv[0] < 0: return chain^ - var s1 = d.accel_nib_states[pv[0]] + var s1 = d.accel.nib_states[pv[0]] var cur = d.table[s1 * 256 + pv[1]] var cap = 4 while cap > 0 and cur >= 0: @@ -2573,9 +2491,9 @@ def pivot_first_candidate[ """ comptime pv = _pivot_prefilter(d) comptime assert pv[0] >= 0 - comptime pk = d.accel_nib_kind[pv[0]] - comptime pt0 = nibble_table_from(d.accel_nib_t0, pv[0]) - comptime pt1 = nibble_table_from(d.accel_nib_t1, pv[0]) + comptime pk = d.accel.nib_kind[pv[0]] + comptime pt0 = nibble_table_from(d.accel.nib_t0, pv[0]) + comptime pt1 = nibble_table_from(d.accel.nib_t1, pv[0]) comptime pivot_byte = UInt8(pv[1]) comptime fchain = _pivot_forced_chain(d, pv) var input_len = len(input) diff --git a/emberregex/static_rdfa.mojo b/emberregex/static_rdfa.mojo index 4876646..1ff3346 100644 --- a/emberregex/static_rdfa.mojo +++ b/emberregex/static_rdfa.mojo @@ -52,15 +52,11 @@ from .constants import CHAR_NEWLINE, is_word_byte from .nfa import NFA, NFAStateKind from .set_reverse import _reverse_edges, _rev_flat_closure from .simd_kernels import ( - ACCEL_SHUFTI, - ACCEL_TRUFFLE, - HAS_FAST_BYTE_SHUFFLE, + AccelSet, _class_contains, - build_shufti_masks, - build_truffle_masks, + accel_exits, nibble_table_from, rfind_in_class, - shufti_encodable, ) from .simd_scan import lane_bits, last_lane_index from .static_bytes import table_bytes @@ -120,13 +116,7 @@ struct RDFA(Copyable, Movable): # or on a nibble-encodable set — the same two flavours as EagerDFA, # scanning backward. Every reverse self-loop is a genuine one (there # is no restart here), so none is vetoed. - var accel_states: List[Int] - var accel_exit1: List[Int] - var accel_exit2: List[Int] # or -1 - var accel_nib_states: List[Int] - var accel_nib_kind: List[Int] - var accel_nib_t0: List[Int] - var accel_nib_t1: List[Int] + var accel: AccelSet def __init__(out self): self.valid = False @@ -140,13 +130,7 @@ struct RDFA(Copyable, Movable): self.any_bol0 = False self.any_bolnl = False self.any_wb = False - self.accel_states = List[Int]() - self.accel_exit1 = List[Int]() - self.accel_exit2 = List[Int]() - self.accel_nib_states = List[Int]() - self.accel_nib_kind = List[Int]() - self.accel_nib_t0 = List[Int]() - self.accel_nib_t1 = List[Int]() + self.accel = AccelSet() def _rev_bol_reaches_start( @@ -705,33 +689,11 @@ def build_reverse_dfa(nfa: NFA, enabled: Bool) -> RDFA: != 0 ): continue - var row = rows.unsafe_get(si) - var exit_count = 0 - for byte in range(256): - if Int(row[byte]) != si: - exit_count += 1 - if exit_count == 0 or exit_count == 256: - continue # never exits / never self-loops: nothing to skip - var exits = List[Int]() - for byte in range(256): - if Int(row[byte]) != si: - exits.append(byte) - if len(exits) <= 2: - result.accel_states.append(si) - result.accel_exit1.append(exits[0]) - result.accel_exit2.append(exits[1] if len(exits) == 2 else -1) - elif HAS_FAST_BYTE_SHUFFLE: - var t0 = List[Int]() - var t1 = List[Int]() - if shufti_encodable(exits): - build_shufti_masks(exits, t0, t1) - result.accel_nib_kind.append(ACCEL_SHUFTI) - else: - build_truffle_masks(exits, t0, t1) - result.accel_nib_kind.append(ACCEL_TRUFFLE) - result.accel_nib_states.append(si) - result.accel_nib_t0.extend(t0^) - result.accel_nib_t1.extend(t1^) + var exits = accel_exits( + rows.unsafe_get(si).ne(SIMD[DType.int32, 256](Int32(si))) + ) + if len(exits) > 0: + result.accel.add(si, exits) result.valid = True result.num_states = nfinal result.table = table^ @@ -781,21 +743,6 @@ def _rfind_exit2[ return floor -def _rdfa_accel_mask_word(d: RDFA, word: Int) -> UInt64: - var m = UInt64(0) - for s in d.accel_states: - if s >> 6 == word: - m |= UInt64(1) << UInt64(s & 63) - for s in d.accel_nib_states: - if s >> 6 == word: - m |= UInt64(1) << UInt64(s & 63) - return m - - -def _rdfa_has_accel(d: RDFA) -> Bool: - return len(d.accel_states) > 0 or len(d.accel_nib_states) > 0 - - @always_inline def _rdfa_accel_skip[ origin: Origin, //, d: RDFA @@ -806,8 +753,8 @@ def _rdfa_accel_skip[ comptime W = simd_width_of[DType.uint8]() if pos - floor < W: return pos - comptime m0 = _rdfa_accel_mask_word(d, 0) - comptime m1 = _rdfa_accel_mask_word(d, 1) + comptime m0 = d.accel.mask_word(0) + comptime m1 = d.accel.mask_word(1) comptime if d.num_states <= 64: if (m0 >> UInt64(cur)) & 1 == 0: return pos @@ -816,19 +763,19 @@ def _rdfa_accel_skip[ if (m >> UInt64(cur & 63)) & 1 == 0: return pos var p = pos - comptime for ai in range(len(d.accel_states)): - comptime a_state = d.accel_states[ai] - comptime a_e1 = UInt8(d.accel_exit1[ai]) + comptime for ai in range(len(d.accel.states)): + comptime a_state = d.accel.states[ai] + comptime a_e1 = UInt8(d.accel.exit1[ai]) comptime a_e2 = UInt8( - d.accel_exit2[ai] if d.accel_exit2[ai] >= 0 else d.accel_exit1[ai] + d.accel.exit2[ai] if d.accel.exit2[ai] >= 0 else d.accel.exit1[ai] ) if cur == a_state: p = _rfind_exit2[e1=a_e1, e2=a_e2](input, p, floor) - comptime for ai in range(len(d.accel_nib_states)): - comptime a_state = d.accel_nib_states[ai] - comptime a_kind = d.accel_nib_kind[ai] - comptime a_t0 = nibble_table_from(d.accel_nib_t0, ai) - comptime a_t1 = nibble_table_from(d.accel_nib_t1, ai) + comptime for ai in range(len(d.accel.nib_states)): + comptime a_state = d.accel.nib_states[ai] + comptime a_kind = d.accel.nib_kind[ai] + comptime a_t0 = nibble_table_from(d.accel.nib_t0, ai) + comptime a_t1 = nibble_table_from(d.accel.nib_t1, ai) if cur == a_state: # Scalar peek at the byte about to be consumed: only # vectorize when it actually self-loops. @@ -878,7 +825,7 @@ def rdfa_find_start[ var best = -1 while True: var f = flg.unsafe_get(cur) - comptime if _rdfa_has_accel(d): + comptime if d.accel.any(): pos = _rdfa_accel_skip[d=d](input, cur, pos, floor) if (f & RDFA_NORM) != 0: best = pos diff --git a/test/test_bench_coverage.mojo b/test/test_bench_coverage.mojo index b603a8d..e8d4a94 100644 --- a/test/test_bench_coverage.mojo +++ b/test/test_bench_coverage.mojo @@ -539,7 +539,7 @@ def test_bench_match_single_byte_run() raises: # stays accelerated (a single-byte self-loop is a genuine run). comptime S = Regex["a+e|x"] assert_true(S._strategy.use_eager_dfa) - comptime n_accel = len(S._edfa.accel_states) + len(S._edfa.accel_nib_states) + comptime n_accel = len(S._edfa.accel.states) + len(S._edfa.accel.nib_states) assert_true(n_accel >= 1) var re = S() var input = "a" * 20480 + "e" diff --git a/test/test_dfa_builders_runtime.mojo b/test/test_dfa_builders_runtime.mojo index 5af4add..512f58e 100644 --- a/test/test_dfa_builders_runtime.mojo +++ b/test/test_dfa_builders_runtime.mojo @@ -342,52 +342,52 @@ def test_reverse_dfa_reuses_states_and_accelerates_runs() raises: assert_equal(_step(loop, 2, "b"), EDFA_DEAD) assert_equal(_find_start(loop, "xaab", 4), 1) assert_equal(_find_start(loop, "xaab", 4, floor=2), 2) - assert_equal(len(loop.accel_states), 0) + assert_equal(len(loop.accel.states), 0) # `.*x`: the state after `x` self-loops on everything but '\n' — one # exit byte; `[^ab]*x` two. var dot = _rdfa(".*x") assert_equal(dot.num_states, 2) assert_equal(dot.flags, [0, Int(RDFA_NORM)]) - assert_equal(dot.accel_states, [1]) - assert_equal(dot.accel_exit1, [Int(CHAR_NEWLINE)]) - assert_equal(dot.accel_exit2, [-1]) - assert_equal(len(dot.accel_nib_states), 0) + assert_equal(dot.accel.states, [1]) + assert_equal(dot.accel.exit1, [Int(CHAR_NEWLINE)]) + assert_equal(dot.accel.exit2, [-1]) + assert_equal(len(dot.accel.nib_states), 0) assert_equal(_find_start(dot, "ab\ncdx", 6), 3) assert_equal(_find_start(dot, "abx", 3), 0) var two = _rdfa("[^ab]*x") - assert_equal(two.accel_states, [1]) - assert_equal(two.accel_exit1, [ord("a")]) - assert_equal(two.accel_exit2, [ord("b")]) + assert_equal(two.accel.states, [1]) + assert_equal(two.accel.exit1, [ord("a")]) + assert_equal(two.accel.exit2, [ord("b")]) assert_equal(_find_start(two, "xbyyx", 5), 2) # A state whose acceptance depends on the byte about to be consumed # (here a BOL flag) is never accelerated, self-loop or not. var bol = _rdfa("^.*x") assert_equal(bol.num_states, 2) assert_equal(bol.flags, [0, Int(RDFA_BOL0)]) - assert_equal(len(bol.accel_states), 0) - assert_equal(len(bol.accel_nib_states), 0) + assert_equal(len(bol.accel.states), 0) + assert_equal(len(bol.accel.nib_states), 0) assert_equal(_find_start(bol, "ab\ncdx", 6), -1) assert_equal(_find_start(bol, "abx", 3), 0) comptime if HAS_FAST_BYTE_SHUFFLE: - assert_equal(loop.accel_nib_states, [2]) - assert_equal(loop.accel_nib_kind, [ACCEL_TRUFFLE]) - assert_equal(len(loop.accel_nib_t0), NIBBLE_TABLE_SIZE) - assert_equal(len(loop.accel_nib_t1), NIBBLE_TABLE_SIZE) + assert_equal(loop.accel.nib_states, [2]) + assert_equal(loop.accel.nib_kind, [ACCEL_TRUFFLE]) + assert_equal(len(loop.accel.nib_t0), NIBBLE_TABLE_SIZE) + assert_equal(len(loop.accel.nib_t1), NIBBLE_TABLE_SIZE) # Three exits in one high nibble: shufti. `[\x01-\x7f]*x` exits # on 0x00 and 0x80-0xFF — nine high nibbles — so truffle. var sh = _rdfa("[^abc]*x") - assert_equal(len(sh.accel_states), 0) - assert_equal(sh.accel_nib_states, [1]) - assert_equal(sh.accel_nib_kind, [ACCEL_SHUFTI]) + assert_equal(len(sh.accel.states), 0) + assert_equal(sh.accel.nib_states, [1]) + assert_equal(sh.accel.nib_kind, [ACCEL_SHUFTI]) assert_equal(_find_start(sh, "cxyyx", 5), 1) var tr = _rdfa("[\\x01-\\x7f]*x") - assert_equal(len(tr.accel_states), 0) - assert_equal(tr.accel_nib_states, [1]) - assert_equal(tr.accel_nib_kind, [ACCEL_TRUFFLE]) + assert_equal(len(tr.accel.states), 0) + assert_equal(tr.accel.nib_states, [1]) + assert_equal(tr.accel.nib_kind, [ACCEL_TRUFFLE]) assert_equal(_find_start(tr, "\nxyyx", 5), 0) else: - assert_equal(len(loop.accel_nib_states), 0) - assert_equal(len(_rdfa("[^abc]*x").accel_nib_states), 0) + assert_equal(len(loop.accel.nib_states), 0) + assert_equal(len(_rdfa("[^abc]*x").accel.nib_states), 0) def test_reverse_dfa_minimize_merges_and_crosses_64_states() raises: diff --git a/test/test_leftmost_first_dfa.mojo b/test/test_leftmost_first_dfa.mojo index 5234bb3..af2ee2f 100644 --- a/test/test_leftmost_first_dfa.mojo +++ b/test/test_leftmost_first_dfa.mojo @@ -136,8 +136,8 @@ def test_spurious_self_loops_are_not_accelerated() raises: "|lap|lab|mop|mob|net|nap|owl|oak|pin|pit|rat|rib|sun|sit|tap|[0-9]{3}" ] assert_true(A._use_lf_dfa) - comptime a_accel = len(A._lfdfa.d.accel_states) + len( - A._lfdfa.d.accel_nib_states + comptime a_accel = len(A._lfdfa.d.accel.states) + len( + A._lfdfa.d.accel.nib_states ) assert_equal(a_accel, 1) # A genuine single-byte loop (the `a+` run) IS accelerated, on both @@ -145,11 +145,11 @@ def test_spurious_self_loops_are_not_accelerated() raises: # 20 KB run measured 16x slower when a loop-set threshold dropped it. comptime B = Regex["a+e|x"] assert_true(B._use_lf_dfa) - comptime b_lf_accel = len(B._lfdfa.d.accel_states) + len( - B._lfdfa.d.accel_nib_states + comptime b_lf_accel = len(B._lfdfa.d.accel.states) + len( + B._lfdfa.d.accel.nib_states ) - comptime b_classic_accel = len(B._edfa.accel_states) + len( - B._edfa.accel_nib_states + comptime b_classic_accel = len(B._edfa.accel.states) + len( + B._edfa.accel.nib_states ) assert_true(b_lf_accel >= 1) assert_true(b_classic_accel >= 1) @@ -565,7 +565,7 @@ def test_reverse_acceleration_bounds() raises: # longer than a SIMD chunk so the vector path is the one exercised. comptime S = Regex["b|.*x"] assert_true(S._use_lf_dfa) - comptime any_rev_accel = len(S._rdfa.accel_states) > 0 + comptime any_rev_accel = len(S._rdfa.accel.states) > 0 assert_true(any_rev_accel) var re = S() var run = "b" + "a" * 40 + "x" diff --git a/test/test_onepass.mojo b/test/test_onepass.mojo index dde51b9..37a3cdd 100644 --- a/test/test_onepass.mojo +++ b/test/test_onepass.mojo @@ -197,7 +197,7 @@ def test_onepass_walker_acceleration() raises: # the shape gate keeps such patterns off `_use_onepass`. comptime op = build_onepass(_build_static_nfa("(a)([^;]*);(b)"), True) assert_true(op.valid) - comptime accel = len(op.accel.accel_states) + len(op.accel.accel_nib_states) + comptime accel = len(op.accel.accel.states) + len(op.accel.accel.nib_states) assert_true(accel >= 1) comptime TN = onepass_table_len(op) comptime TBL = static_bytes[onepass_table_str[TN](op)]() diff --git a/test/test_set_build_parity.mojo b/test/test_set_build_parity.mojo index 3c8496b..0fb132a 100644 --- a/test/test_set_build_parity.mojo +++ b/test/test_set_build_parity.mojo @@ -49,13 +49,13 @@ def check(pats: List[String]) raises: check_lists(a.nl_len, b.nl_len, "mdfa nl_len") check_lists(a.end_off, b.end_off, "mdfa end_off") check_lists(a.end_len, b.end_len, "mdfa end_len") - check_lists(a.accel_states, b.accel_states, "mdfa accel") - check_lists(a.accel_exit1, b.accel_exit1, "mdfa exit1") - check_lists(a.accel_exit2, b.accel_exit2, "mdfa exit2") - check_lists(a.accel_nib_states, b.accel_nib_states, "mdfa nib states") - check_lists(a.accel_nib_kind, b.accel_nib_kind, "mdfa nib kind") - check_lists(a.accel_nib_t0, b.accel_nib_t0, "mdfa nib t0") - check_lists(a.accel_nib_t1, b.accel_nib_t1, "mdfa nib t1") + check_lists(a.accel.states, b.accel.states, "mdfa accel") + check_lists(a.accel.exit1, b.accel.exit1, "mdfa exit1") + check_lists(a.accel.exit2, b.accel.exit2, "mdfa exit2") + check_lists(a.accel.nib_states, b.accel.nib_states, "mdfa nib states") + check_lists(a.accel.nib_kind, b.accel.nib_kind, "mdfa nib kind") + check_lists(a.accel.nib_t0, b.accel.nib_t0, "mdfa nib t0") + check_lists(a.accel.nib_t1, b.accel.nib_t1, "mdfa nib t1") var ra = _build_reverse_dfa_list(nfa) var rb = build_reverse_dfa(nfa, True) diff --git a/test/test_set_phase2.mojo b/test/test_set_phase2.mojo index 0d279c6..dc7102d 100644 --- a/test/test_set_phase2.mojo +++ b/test/test_set_phase2.mojo @@ -42,10 +42,10 @@ def _mdfa_scan[ def _accelerates(d: MultiDFA, state: Int) -> Bool: """Comptime: is `state` one of the SIMD-scanned self-loop states?""" - for s in d.accel_states: + for s in d.accel.states: if s == state: return True - for s in d.accel_nib_states: + for s in d.accel.nib_states: if s == state: return True return False diff --git a/test/test_simd_kernels.mojo b/test/test_simd_kernels.mojo index dbde724..adcb538 100644 --- a/test/test_simd_kernels.mojo +++ b/test/test_simd_kernels.mojo @@ -142,7 +142,7 @@ def test_find_in_class_start_offset() raises: def test_email_pattern_gets_nibble_accel() raises: comptime E = Regex["[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}"] comptime if HAS_FAST_BYTE_SHUFFLE: - comptime n_nib = len(E._edfa.accel_nib_states) + comptime n_nib = len(E._edfa.accel.nib_states) assert_true(n_nib >= 1) var re = E() var r = re.search("reach me at first.last@example.com or in person") @@ -157,7 +157,7 @@ def test_nibble_accel_long_runs() raises: comptime W = simd_width_of[DType.uint8]() comptime S = Regex["[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}"] comptime if HAS_FAST_BYTE_SHUFFLE: - comptime n_nib = len(S._edfa.accel_nib_states) + comptime n_nib = len(S._edfa.accel.nib_states) assert_true(n_nib >= 1) var re = S() var user = String("u") * (3 * W + 1) @@ -173,7 +173,7 @@ def test_shufti_state_pattern() raises: # `[^a-z]*[a-z]+` self-loops exit on a-z (2 high nibbles): shufti. comptime E = Regex["[^a-z]*[a-z]+"] comptime if HAS_FAST_BYTE_SHUFFLE: - comptime kinds = E._edfa.accel_nib_kind + comptime kinds = E._edfa.accel.nib_kind comptime has_shufti = ACCEL_SHUFTI in kinds assert_true(has_shufti) var re = E() @@ -190,7 +190,7 @@ def test_nibble_accel_high_bytes() raises: # regions exercise the high-nibble table half. comptime S = Regex["[^a-z]*[a-z]+"] comptime if HAS_FAST_BYTE_SHUFFLE: - comptime n_nib = len(S._edfa.accel_nib_states) + comptime n_nib = len(S._edfa.accel.nib_states) assert_true(n_nib >= 1) var re = S() var buf = List[Byte]() @@ -210,7 +210,7 @@ def test_nibble_accel_high_bytes() raises: def test_dotstar_suffix_still_accelerated() raises: # `.*x` keeps its 2-exit-byte compare path alongside nibble accel. comptime E = Regex[".*x"] - comptime n_exit2 = len(E._edfa.accel_states) + comptime n_exit2 = len(E._edfa.accel.states) assert_true(n_exit2 >= 1) var re = E() comptime W = simd_width_of[DType.uint8]() @@ -226,7 +226,7 @@ def test_nibble_accel_findall_multiline() raises: # Accel must not skip past '\n' boundaries that end matches. comptime S = Regex["[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}"] comptime if HAS_FAST_BYTE_SHUFFLE: - comptime n_nib = len(S._edfa.accel_nib_states) + comptime n_nib = len(S._edfa.accel.nib_states) assert_true(n_nib >= 1) var re = S() var text = "a@b.com\nnope\nlong.user@sub.host.org\n" From 3f9f43a7cebeb729e6bfefe7406b15ddb534916b Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:13:20 -0600 Subject: [PATCH 35/65] cleanup: fold group_str(String) and the 3-arg _sbt_match_at into their siblings Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/engine.mojo | 15 ++------------- emberregex/result.mojo | 13 ++----------- 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/emberregex/engine.mojo b/emberregex/engine.mojo index 0716d94..e58810a 100644 --- a/emberregex/engine.mojo +++ b/emberregex/engine.mojo @@ -1723,20 +1723,9 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( def _sbt_match_at[ origin: Origin, // ](self, input: Span[Byte, origin], start: Int) -> Int: - """`_sbt_match_at` discarding the capture slots.""" + """`_sbt_match_at` discarding the capture slots, with a fresh + LF_SBT_ATTEMPT_BUDGET.""" var slots = materialize[ALL_NEG_ONES[Self._num_slots]]() - return self._sbt_match_at(input, start, slots) - - @always_inline - def _sbt_match_at[ - origin: Origin, // - ]( - self, - input: Span[Byte, origin], - start: Int, - mut slots: Array[Int, Self._num_slots], - ) -> Int: - """`_sbt_match_at` with a fresh LF_SBT_ATTEMPT_BUDGET.""" var budget = LF_SBT_ATTEMPT_BUDGET return self._sbt_match_at(input, start, slots, budget) diff --git a/emberregex/result.mojo b/emberregex/result.mojo index f94cd6c..b34816d 100644 --- a/emberregex/result.mojo +++ b/emberregex/result.mojo @@ -66,17 +66,8 @@ struct MatchResult[num_slots: Int](Copyable, Movable, Writable): return self.slots[2 * index - 2] != -1 def group_str(self, input: String, index: Int) -> String: - """Extract the text matched by capture group `index` (1-based). - - Returns empty string if the group didn't match. - """ - if index < 1 or index > Self.group_count or not self.matched: - return "" - var s = self.slots[2 * index - 2] - var e = self.slots[2 * index - 1] - if s == -1 or e == -1: - return "" - return String(unsafe_from_utf8=input.as_bytes()[s:e]) + """`group_str` over `input`'s bytes.""" + return self.group_str(input.as_bytes(), index) def group_str[ origin: Origin, // From 3b3002e535a066a3bfad24432053a8d3dd5b84fb Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:14:25 -0600 Subject: [PATCH 36/65] cleanup: call table_bytes directly instead of edfa/rdfa_table_str Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 2 +- emberregex/engine.mojo | 16 +++++++--------- emberregex/static_dfa.mojo | 13 ------------- emberregex/static_rdfa.mojo | 9 --------- test/test_leftmost_first_dfa.mojo | 5 ++--- test/test_word_boundary_dfa.mojo | 10 ++++------ 6 files changed, 14 insertions(+), 41 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index cf851e7..ba2c7ca 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -162,7 +162,7 @@ loops at 35-70 us per element op, spread over every lane. Rules: Int64]().unsafe_load[width=256]()` / `.unsafe_store(vec)` (a `List[Int]` holds 64-bit lanes). `List(fill=, length=)` + vector stores replaced 256 appends per state and the per-cell `Array` copies - (`_edfa_finish`, `edfa_table_str`, `sheng_masks_str`, the set lanes). + (`_edfa_finish`, `table_bytes`, `sheng_masks_str`, the set lanes). - **A materialized table is a string literal, never an `Array`.** A comptime `Array` a walker `materialize`s becomes a global whose LLVM initializer is folded one `insertvalue` per element — O(n²): a diff --git a/emberregex/engine.mojo b/emberregex/engine.mojo index 6c31855..7bc885f 100644 --- a/emberregex/engine.mojo +++ b/emberregex/engine.mojo @@ -86,7 +86,6 @@ from .static_dfa import ( _wb_cont_reaches_bol, _pivot_prefilter, build_eager_dfa, - edfa_table_str, edfa_table_len, edfa_flags_arr, edfa_id_dtype, @@ -103,9 +102,8 @@ from .static_rdfa import ( build_reverse_dfa, rdfa_find_start, rdfa_flags_arr, - rdfa_table_str, ) -from .static_bytes import static_bytes +from .static_bytes import static_bytes, table_bytes from .sheng import ( sheng_cap_for, sheng_full_match, @@ -1355,8 +1353,8 @@ struct Regex[pattern: String](Copyable, Movable): # of a per-call stack copy (see edfa_table_len). comptime _EDFA_TN = edfa_table_len(Self._edfa.num_states) comptime _EDFA_DT = edfa_id_dtype(Self._edfa.num_states) - comptime _EDFA_TABLE_S = edfa_table_str[Self._EDFA_TN, Self._EDFA_DT]( - Self._edfa + comptime _EDFA_TABLE_S = table_bytes[Self._EDFA_DT]( + Self._edfa.table, Self._EDFA_TN ) comptime _EDFA_FLAGS = edfa_flags_arr[Self._edfa.num_states](Self._edfa) comptime _EDFA_TABLE = static_bytes[Self._EDFA_TABLE_S]() @@ -1370,8 +1368,8 @@ struct Regex[pattern: String](Copyable, Movable): comptime _SHENG_MASKS = static_bytes[Self._SHENG_MASKS_S]() comptime _LFDFA_TN = edfa_table_len(Self._lfdfa.d.num_states) comptime _LFDFA_DT = edfa_id_dtype(Self._lfdfa.d.num_states) - comptime _LFDFA_TABLE_S = edfa_table_str[Self._LFDFA_TN, Self._LFDFA_DT]( - Self._lfdfa.d + comptime _LFDFA_TABLE_S = table_bytes[Self._LFDFA_DT]( + Self._lfdfa.d.table, Self._LFDFA_TN ) comptime _LFDFA_FLAGS = edfa_flags_arr[Self._lfdfa.d.num_states]( Self._lfdfa.d @@ -1384,8 +1382,8 @@ struct Regex[pattern: String](Copyable, Movable): comptime _LF_SHENG_MASKS = static_bytes[Self._LF_SHENG_MASKS_S]() comptime _RDFA_TN = edfa_table_len(Self._rdfa.num_states) comptime _RDFA_DT = edfa_id_dtype(Self._rdfa.num_states) - comptime _RDFA_TABLE_S = rdfa_table_str[Self._RDFA_TN, Self._RDFA_DT]( - Self._rdfa + comptime _RDFA_TABLE_S = table_bytes[Self._RDFA_DT]( + Self._rdfa.table, Self._RDFA_TN ) comptime _RDFA_FLAGS = rdfa_flags_arr[Self._rdfa.num_states](Self._rdfa) comptime _RDFA_TABLE = static_bytes[Self._RDFA_TABLE_S]() diff --git a/emberregex/static_dfa.mojo b/emberregex/static_dfa.mojo index 1363815..b2969e5 100644 --- a/emberregex/static_dfa.mojo +++ b/emberregex/static_dfa.mojo @@ -20,7 +20,6 @@ from .ast import AnchorKind from .constants import CHAR_NEWLINE, is_word_byte from .nfa import NFA, NFAStateKind from .optimize import PROBE_RANKS -from .static_bytes import table_bytes from .dfa import _reaches_match from .charset import BITMAP_WIDTH from .simd_scan import first_lane_index, lane_bits, simd_find_byte @@ -1841,18 +1840,6 @@ def _edfa_finish( return pstarts^ -def edfa_table_str[n: Int, dt: DType](d: EagerDFA) -> String: - """Comptime: the flat table as `n` little-endian `dt` entries (see - static_bytes.mojo for why a string, not an Array). - - `dt` comes from `edfa_id_dtype`, `n` from `edfa_table_len` (it may - exceed the table: the tail stays EDFA_DEAD padding); EDFA_DEAD (-1) - survives the narrowing, so the walkers keep their sign-bit dead test. - """ - assert n == 0 or n >= len(d.table), "table string shorter than the table" - return table_bytes[dt](d.table, n) - - def edfa_flags_arr[n: Int](d: EagerDFA) -> Array[UInt8, n]: """Comptime conversion of per-state flags to a materializable array.""" var arr = Array[UInt8, n](fill=0) diff --git a/emberregex/static_rdfa.mojo b/emberregex/static_rdfa.mojo index 835a223..56b1daa 100644 --- a/emberregex/static_rdfa.mojo +++ b/emberregex/static_rdfa.mojo @@ -63,7 +63,6 @@ from .simd_kernels import ( shufti_encodable, ) from .simd_scan import lane_bits, last_lane_index -from .static_bytes import table_bytes from .static_dfa import ( edfa_id_dtype, EDFA_DEAD, @@ -744,14 +743,6 @@ def build_reverse_dfa(nfa: NFA, enabled: Bool) -> RDFA: return result^ -def rdfa_table_str[n: Int, dt: DType](d: RDFA) -> String: - """Comptime: the flat table as `n` little-endian `dt` entries (narrow id - type from `edfa_id_dtype`, `n` from `edfa_table_len`; EDFA_DEAD - survives). See static_bytes.mojo for why a string.""" - assert n == 0 or n >= len(d.table), "table string shorter than the table" - return table_bytes[dt](d.table, n) - - def rdfa_flags_arr[n: Int](d: RDFA) -> Array[UInt8, n]: var arr = Array[UInt8, n](fill=0) for i in range(n): diff --git a/test/test_leftmost_first_dfa.mojo b/test/test_leftmost_first_dfa.mojo index 254bffa..e398519 100644 --- a/test/test_leftmost_first_dfa.mojo +++ b/test/test_leftmost_first_dfa.mojo @@ -13,13 +13,12 @@ newlines and bytes >= 0x80. from emberregex import Regex from emberregex.simd_kernels import HAS_FAST_BYTE_SHUFFLE -from emberregex.static_bytes import static_bytes +from emberregex.static_bytes import static_bytes, table_bytes from emberregex.static_dfa import ( EDFA_TABLE_MIN_BYTES, EagerDFA, edfa_flags_arr, edfa_id_dtype, - edfa_table_str, ) from emberregex.static_rdfa import rdfa_find_start from emberregex.static_lfdfa import ( @@ -690,7 +689,7 @@ def test_wide_list_signature_renumbering() raises: assert_true(lf.valid) comptime tn = lf.d.num_states * 256 comptime dt = edfa_id_dtype(lf.d.num_states) - comptime table = static_bytes[edfa_table_str[tn, dt](lf.d)]() + comptime table = static_bytes[table_bytes[dt](lf.d.table, tn)]() comptime flags = edfa_flags_arr[lf.d.num_states](lf.d) var input = String("!!!!!!!!!!!!!!!!!!!!q!!") var bytes = input.as_bytes() diff --git a/test/test_word_boundary_dfa.mojo b/test/test_word_boundary_dfa.mojo index bebc8ec..2e25618 100644 --- a/test/test_word_boundary_dfa.mojo +++ b/test/test_word_boundary_dfa.mojo @@ -13,7 +13,7 @@ shapes engine selection leaves on the backtracker. """ from emberregex import Regex -from emberregex.static_bytes import static_bytes +from emberregex.static_bytes import static_bytes, table_bytes from emberregex.static_dfa import ( EDFA_MATCH_IF_NONWORD, EDFA_MATCH_IF_WORD, @@ -25,14 +25,12 @@ from emberregex.static_dfa import ( edfa_full_match, edfa_id_dtype, edfa_match_at, - edfa_table_str, ) from emberregex.static_lfdfa import LFDFA, build_lf_dfa, lfdfa_find_end from emberregex.static_rdfa import ( build_reverse_dfa, rdfa_find_start, rdfa_flags_arr, - rdfa_table_str, ) from std.testing import assert_true, assert_false, assert_equal, TestSuite @@ -239,19 +237,19 @@ def _forced_lane_check[p: StaticString](input: String, label: String) raises: comptime assert ed.valid comptime ETN = ed.num_states * 256 comptime EDT = edfa_id_dtype(ed.num_states) - comptime etbl = static_bytes[edfa_table_str[ETN, EDT](ed)]() + comptime etbl = static_bytes[table_bytes[EDT](ed.table, ETN)]() comptime efl = edfa_flags_arr[ed.num_states](ed) comptime lf = build_lf_dfa(nfa, True) comptime assert lf.valid comptime LTN = lf.d.num_states * 256 comptime LDT = edfa_id_dtype(lf.d.num_states) - comptime ltbl = static_bytes[edfa_table_str[LTN, LDT](lf.d)]() + comptime ltbl = static_bytes[table_bytes[LDT](lf.d.table, LTN)]() comptime lfl = edfa_flags_arr[lf.d.num_states](lf.d) comptime rd = build_reverse_dfa(nfa, True) comptime assert rd.valid comptime RTN = rd.num_states * 256 comptime RDT = edfa_id_dtype(rd.num_states) - comptime rtbl = static_bytes[rdfa_table_str[RTN, RDT](rd)]() + comptime rtbl = static_bytes[table_bytes[RDT](rd.table, RTN)]() comptime rfl = rdfa_flags_arr[rd.num_states](rd) var re = Regex[p]() From 7a122ba31f0dea0511cb1689e4270b4ff60f2b8d Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:15:43 -0600 Subject: [PATCH 37/65] cleanup: findall/finditer backtracker loops share one shape _pike_findall reuses _findall_append (which loses its always-equal `n` parameter and a redundant group-count test); the BOL_MULTILINE loops of _findall_impl/_finditer_impl drop the matched branch's duplicate newline scan (the DFA-lane copies already use the short form); both chains end in `else:` without the dead trailing return. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/engine.mojo | 48 +++++++++--------------------------------- 1 file changed, 10 insertions(+), 38 deletions(-) diff --git a/emberregex/engine.mojo b/emberregex/engine.mojo index e58810a..95713f0 100644 --- a/emberregex/engine.mojo +++ b/emberregex/engine.mojo @@ -2884,12 +2884,6 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( and input_bytes.unsafe_get(pos - 1) == CHAR_NEWLINE ): continue - # Otherwise skip to the next BOL - var nl = simd_find_byte(input_bytes, CHAR_NEWLINE, pos) - if nl < 0: - break - pos = nl + 1 - continue # Skip to next BOL position var nl = simd_find_byte(input_bytes, CHAR_NEWLINE, pos) if nl < 0: @@ -2897,7 +2891,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( pos = nl + 1 return results^ - elif Self._strategy.start_anchor != AnchorKind.BOL_MULTILINE: + else: var pos = 0 while pos <= input_len: comptime if Self._use_scan_filter: @@ -2920,22 +2914,21 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( else: pos = _scan_bump[Self._is_unicode](input_bytes, pos) return results^ - return results^ @always_inline - def _findall_append[ - n: Int - ]( + def _findall_append( self, mut results: List[String], input: String, pos: Int, end: Int, - slots: Array[Int, n], + slots: Array[Int, Self._num_slots], ): + """findall's per-match string: group 1 when it participated, else + the whole span.""" var input_bytes = input.as_bytes() comptime if Self._num_slots >= 2: - if Self._group_count > 0 and slots[0] >= 0 and slots[1] >= 0: + if slots[0] >= 0 and slots[1] >= 0: results.append( String(unsafe_from_utf8=input_bytes[slots[0] : slots[1]]) ) @@ -2961,23 +2954,9 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( if not result.matched: # Unanchored: a miss covers every start >= pos. break - comptime if Self._group_count > 0: - if result.group_matched(1): - results.append(result.group_str(input_bytes, 1)) - else: - results.append( - String( - unsafe_from_utf8=input_bytes[ - result.start : result.end - ] - ) - ) - else: - results.append( - String( - unsafe_from_utf8=input_bytes[result.start : result.end] - ) - ) + self._findall_append( + results, input, result.start, result.end, result.slots + ) if result.end > result.start: pos = result.end else: @@ -3031,12 +3010,6 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( and input_bytes.unsafe_get(pos - 1) == CHAR_NEWLINE ): continue - # Otherwise skip to the next BOL - var nl = simd_find_byte(input_bytes, CHAR_NEWLINE, pos) - if nl < 0: - break - pos = nl + 1 - continue # Skip to next BOL position var nl = simd_find_byte(input_bytes, CHAR_NEWLINE, pos) if nl < 0: @@ -3044,7 +3017,7 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( pos = nl + 1 return results^ - elif Self._strategy.start_anchor != AnchorKind.BOL_MULTILINE: + else: var pos = 0 while pos <= input_len: comptime if Self._use_scan_filter: @@ -3071,7 +3044,6 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( else: pos = _scan_bump[Self._is_unicode](input_bytes, pos) return results^ - return results^ def replace(mut self, input: String, replacement: String) -> String: """Replace all non-overlapping matches with replacement string. From 125d7e93a17c4f9c7d15700fd9632529eaa8426c Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:17:57 -0600 Subject: [PATCH 38/65] cleanup: one @always_inline giveback helper for simple and counted greedy loops The counted-greedy exit block was the simple loop's with pos replaced by min_pos; both now call _sbt_giveback[body, exit](lo_pos, max_pos). It is inlined into the owning _sbt_try_match state, so it adds no frame and no out-of-line instantiation (the out-of-line walker instantiations emit byte-identical asm). Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/backtrack.mojo | 405 +++++++++++++++++--------------------- 1 file changed, 183 insertions(+), 222 deletions(-) diff --git a/emberregex/backtrack.mojo b/emberregex/backtrack.mojo index 8f3bc0e..674b68f 100644 --- a/emberregex/backtrack.mojo +++ b/emberregex/backtrack.mojo @@ -1275,6 +1275,154 @@ def sbt_memo_rows_of(nfa: NFA) -> Int: return sbt_memo_rows(nfa, _sbt_needs_depth_guard(nfa)) +@always_inline +def _sbt_giveback[ + origin: Origin, + //, + pattern: String, + body_idx: Int, + exit_idx: Int, + num_slots: Int, + anchored_end: Bool, + memo_on: Bool, +]( + input: Span[Byte, origin], + lo_pos: Int, + max_pos: Int, + mut slots: Array[Int, num_slots], + mut budget: Int, + memo_addr: Int, + stack_floor: Int, + end_at: Int, +) -> Int: + """A greedy loop's exit (`exit_idx`), tried from `max_pos` back down to + `lo_pos` once the body (`body_idx`, one consuming state) has eaten + everything it can. Shared by the simple loop (`lo_pos` = its entry) + and the counted greedy chain (`lo_pos` = past its required copies). + + `@always_inline` is load-bearing: this is the body of the one + `_sbt_try_match` state that owns the loop, split out only so both + loop forms share it — it must add no frame and no call to a walk + that can recurse per input byte (see `_sbt_needs_depth_guard`). + + Trivial exits — MATCH directly, or ANCHOR(EOL/EOL_MULTILINE) → MATCH + — are folded in, skipping the recursive call on the success path.""" + comptime nfa = _build_static_nfa(pattern) + comptime exit_is_match = _exit_is_match(nfa, exit_idx) + comptime exit_is_eol_then_match = _exit_is_eol_then_match(nfa, exit_idx) + var input_len = len(input) + comptime if exit_is_match and anchored_end: + # Anchored MATCH accepts only at the target position + # (`end_at`, else end of input); the loop can stop + # anywhere in [lo_pos, max_pos]. + var target = end_at if end_at >= 0 else input_len + if lo_pos <= target and target <= max_pos: + return target + return -1 + elif exit_is_match: + # Greedy `body* MATCH` — max_pos is the longest match. + return max_pos + elif exit_is_eol_then_match and anchored_end: + # With MATCH anchored to the target, success reduces to + # the loop reaching it AND the EOL anchor holding there. + # At end of input the anchor is trivially true, which is + # the only case when `end_at` is unset. + comptime anchored_eol_ml = ( + nfa.states[exit_idx].anchor_type == AnchorKind.EOL_MULTILINE + ) + var target = end_at if end_at >= 0 else input_len + if lo_pos <= target and target <= max_pos: + if target == input_len: + return target + comptime if anchored_eol_ml: + if input.unsafe_get(target) == CHAR_NEWLINE: + return target + return -1 + elif exit_is_eol_then_match: + # Greedy `body* ANCHOR(EOL/EOL_MULTILINE) MATCH` — fold + # the anchor check into the loop so we don't recurse for + # every position checked. + comptime is_multiline_eol = ( + nfa.states[exit_idx].anchor_type == AnchorKind.EOL_MULTILINE + ) + var p = max_pos + while p >= lo_pos: + comptime if is_multiline_eol: + if p == input_len or input.unsafe_get(p) == CHAR_NEWLINE: + return p + else: + if p == input_len: + return p + p -= 1 + return -1 + else: + # General exit: hand bytes back one at a time. Every + # position in [lo_pos, max_pos) holds a byte the body ate, + # so an exit that cannot START on such a byte fails + # there without being run (auto-possessification). Scoped to + # this arm: `_sbt_loop_filter` walks the NFA through + # `first_byte_bitmap_of`, and the folded forms never read it. + comptime lf = _sbt_loop_filter(nfa, body_idx, exit_idx) + comptime mode = lf.mode + comptime exit_bits = lf.exit_bits + comptime exit_byte = _sbt_single_byte(exit_bits) + comptime if mode == SBT_GIVEBACK_POSSESSIVE: + # Only max_pos can start the exit — and the mode + # also proves the exit consumes a byte, so end of + # input cannot match either. No first-byte test + # here: the exit's own first state runs exactly + # that test, specialized, one call deeper. + if max_pos < input_len: + return _sbt_try_match[ + pattern=pattern, + state_idx=exit_idx, + num_slots=num_slots, + anchored_end=anchored_end, + memo_on=memo_on, + ]( + input, + max_pos, + slots, + budget, + memo_addr, + stack_floor, + end_at, + ) + return -1 + else: + var p = max_pos + while p >= lo_pos: + if budget < 0: + return -1 + comptime if mode == SBT_GIVEBACK_FILTER: + # p == input_len is skipped too: the exit + # needs a byte and there is none left. + if p >= input_len or not _sbt_first_byte_test[ + exit_bits, exit_byte + ](input.unsafe_get(p)): + p -= 1 + continue + var result = _sbt_try_match[ + pattern=pattern, + state_idx=exit_idx, + num_slots=num_slots, + anchored_end=anchored_end, + memo_on=memo_on, + ]( + input, + p, + slots, + budget, + memo_addr, + stack_floor, + end_at, + ) + if result >= 0: + return result + p -= 1 + return -1 + + def _sbt_try_match[ origin: Origin, //, @@ -1418,110 +1566,25 @@ def _sbt_try_match[ max_pos += 1 if max_pos < min_pos: return -1 - # The folded-exit forms below are the simple loop's, with - # `pos` replaced by `min_pos`: the chain may not hand back - # past its required copies. - comptime exit_is_match = _exit_is_match(nfa, cexit) - comptime exit_is_eol_then_match = _exit_is_eol_then_match( - nfa, cexit + # The simple loop's giveback, floored at `min_pos`: the + # chain may not hand back past its required copies. + return _sbt_giveback[ + pattern=pattern, + body_idx=counted.body, + exit_idx=cexit, + num_slots=num_slots, + anchored_end=anchored_end, + memo_on=memo_on, + ]( + input, + min_pos, + max_pos, + slots, + budget, + memo_addr, + stack_floor, + end_at, ) - comptime if exit_is_match and anchored_end: - var target = end_at if end_at >= 0 else input_len - if min_pos <= target and target <= max_pos: - return target - return -1 - elif exit_is_match: - return max_pos - elif exit_is_eol_then_match and anchored_end: - comptime a_eol_ml = ( - nfa.states[cexit].anchor_type - == AnchorKind.EOL_MULTILINE - ) - var target = end_at if end_at >= 0 else input_len - if min_pos <= target and target <= max_pos: - if target == input_len: - return target - comptime if a_eol_ml: - if input.unsafe_get(target) == CHAR_NEWLINE: - return target - return -1 - elif exit_is_eol_then_match: - comptime is_ml_eol = ( - nfa.states[cexit].anchor_type - == AnchorKind.EOL_MULTILINE - ) - var p = max_pos - while p >= min_pos: - comptime if is_ml_eol: - if ( - p == input_len - or input.unsafe_get(p) == CHAR_NEWLINE - ): - return p - else: - if p == input_len: - return p - p -= 1 - return -1 - else: - # Scoped here, not above: `_sbt_loop_filter` walks the - # NFA through `first_byte_bitmap_of`, and the folded - # forms above never read it. - comptime lf = _sbt_loop_filter(nfa, counted.body, cexit) - comptime mode = lf.mode - comptime exit_bits = lf.exit_bits - comptime exit_byte = _sbt_single_byte(exit_bits) - comptime if mode == SBT_GIVEBACK_POSSESSIVE: - # Nothing in [min_pos, max_pos) can start the exit, - # and the mode also proves the exit consumes a - # byte, so end of input cannot match either. - if max_pos < input_len: - return _sbt_try_match[ - pattern=pattern, - state_idx=cexit, - num_slots=num_slots, - anchored_end=anchored_end, - memo_on=memo_on, - ]( - input, - max_pos, - slots, - budget, - memo_addr, - stack_floor, - end_at, - ) - return -1 - else: - var p = max_pos - while p >= min_pos: - if budget < 0: - return -1 - comptime if mode == SBT_GIVEBACK_FILTER: - if p >= input_len or not _sbt_first_byte_test[ - exit_bits, exit_byte - ](input.unsafe_get(p)): - p -= 1 - continue - var result = _sbt_try_match[ - pattern=pattern, - state_idx=cexit, - num_slots=num_slots, - anchored_end=anchored_end, - memo_on=memo_on, - ]( - input, - p, - slots, - budget, - memo_addr, - stack_floor, - end_at, - ) - if result >= 0: - return result - p -= 1 - return -1 else: # Lazy: take the required copies, then try the exit after # each further copy — shortest count first, the mirror of @@ -1674,14 +1737,6 @@ def _sbt_try_match[ # Greedy: scan forward consuming as many chars as possible, # then try the exit (out2) from rightmost to leftmost position. comptime body = nfa.states[out1] - # Detect trivial exits — out2 is MATCH directly or - # ANCHOR(EOL/EOL_MULTILINE) → MATCH. In both cases the loop - # body can fold the exit check inline and skip the recursive - # _sbt_try_match call entirely on the success path. - comptime exit_is_match = _exit_is_match(nfa, out2) - comptime exit_is_eol_then_match = _exit_is_eol_then_match( - nfa, out2 - ) var input_len = len(input) var max_pos = pos comptime if body.kind == NFAStateKind.ANY: @@ -1705,117 +1760,23 @@ def _sbt_try_match[ bitmap, negated, UInt32(input.unsafe_get(max_pos)) ): max_pos += 1 - comptime if exit_is_match and anchored_end: - # Anchored MATCH accepts only at the target position - # (`end_at`, else end of input); the loop can stop - # anywhere in [pos, max_pos]. - var target = end_at if end_at >= 0 else input_len - if pos <= target and target <= max_pos: - return target - return -1 - elif exit_is_match: - # Greedy `body* MATCH` — max_pos is the longest match. - return max_pos - elif exit_is_eol_then_match and anchored_end: - # With MATCH anchored to the target, success reduces to - # the loop reaching it AND the EOL anchor holding there. - # At end of input the anchor is trivially true, which is - # the only case when `end_at` is unset. - comptime anchored_eol_ml = ( - nfa.states[out2].anchor_type == AnchorKind.EOL_MULTILINE - ) - var target = end_at if end_at >= 0 else input_len - if pos <= target and target <= max_pos: - if target == input_len: - return target - comptime if anchored_eol_ml: - if input.unsafe_get(target) == CHAR_NEWLINE: - return target - return -1 - elif exit_is_eol_then_match: - # Greedy `body* ANCHOR(EOL/EOL_MULTILINE) MATCH` — fold - # the anchor check into the loop so we don't recurse for - # every position checked. - comptime is_multiline_eol = ( - nfa.states[out2].anchor_type == AnchorKind.EOL_MULTILINE - ) - var p = max_pos - while p >= pos: - comptime if is_multiline_eol: - if ( - p == input_len - or input.unsafe_get(p) == CHAR_NEWLINE - ): - return p - else: - if p == input_len: - return p - p -= 1 - return -1 - else: - # General exit: hand bytes back one at a time. Every - # position in [pos, max_pos) holds a byte the body ate, - # so an exit that cannot START on such a byte fails - # there without being run (auto-possessification). - comptime lf = _sbt_loop_filter(nfa, out1, out2) - comptime mode = lf.mode - comptime exit_bits = lf.exit_bits - comptime exit_byte = _sbt_single_byte(exit_bits) - comptime if mode == SBT_GIVEBACK_POSSESSIVE: - # Only max_pos can start the exit — and the mode - # also proves the exit consumes a byte, so end of - # input cannot match either. No first-byte test - # here: the exit's own first state runs exactly - # that test, specialized, one call deeper. - if max_pos < input_len: - return _sbt_try_match[ - pattern=pattern, - state_idx=out2, - num_slots=num_slots, - anchored_end=anchored_end, - memo_on=memo_on, - ]( - input, - max_pos, - slots, - budget, - memo_addr, - stack_floor, - end_at, - ) - return -1 - else: - var p = max_pos - while p >= pos: - if budget < 0: - return -1 - comptime if mode == SBT_GIVEBACK_FILTER: - # p == input_len is skipped too: the exit - # needs a byte and there is none left. - if p >= input_len or not _sbt_first_byte_test[ - exit_bits, exit_byte - ](input.unsafe_get(p)): - p -= 1 - continue - var result = _sbt_try_match[ - pattern=pattern, - state_idx=out2, - num_slots=num_slots, - anchored_end=anchored_end, - memo_on=memo_on, - ]( - input, - p, - slots, - budget, - memo_addr, - stack_floor, - end_at, - ) - if result >= 0: - return result - p -= 1 - return -1 + return _sbt_giveback[ + pattern=pattern, + body_idx=out1, + exit_idx=out2, + num_slots=num_slots, + anchored_end=anchored_end, + memo_on=memo_on, + ]( + input, + pos, + max_pos, + slots, + budget, + memo_addr, + stack_floor, + end_at, + ) elif is_simple_lazy: # Lazy: try the exit (out1 — lazy splits prefer it) first, # then consume one body char (out2) and repeat. This is From c1f72bafa8ad2c39263759f989fd255e8a4ca4fb Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:30:25 -0600 Subject: [PATCH 39/65] cleanup: replace one-off List->Array converters with list_arr/int_arr Co-Authored-By: Claude Opus 5.5 (1M context) --- bench/bench_set.mojo | 18 +++---- emberregex/engine.mojo | 41 ++++++++------- emberregex/onepass.mojo | 27 ++-------- emberregex/optimize.mojo | 17 ------ emberregex/set_ac.mojo | 7 --- emberregex/set_bitnfa.mojo | 21 -------- emberregex/set_combine.mojo | 7 --- emberregex/set_dfa.mojo | 7 --- emberregex/set_engine.mojo | 87 ++++++++++++++++--------------- emberregex/set_reverse.mojo | 7 --- emberregex/set_rose.mojo | 18 +------ emberregex/static_bytes.mojo | 25 ++++++++- emberregex/static_dfa.mojo | 22 ++------ emberregex/static_rdfa.mojo | 7 --- test/test_leftmost_first_dfa.mojo | 5 +- test/test_onepass.mojo | 9 ++-- test/test_reverse_literal.mojo | 10 ++-- test/test_set_ac.mojo | 5 +- test/test_set_bench_coverage.mojo | 18 +++---- test/test_set_paths.mojo | 14 +++-- test/test_set_phase2.mojo | 5 +- test/test_set_phase3.mojo | 14 +++-- test/test_word_boundary_dfa.mojo | 10 ++-- 23 files changed, 146 insertions(+), 255 deletions(-) diff --git a/bench/bench_set.mojo b/bench/bench_set.mojo index 54d7e4f..b9a68f5 100644 --- a/bench/bench_set.mojo +++ b/bench/bench_set.mojo @@ -33,20 +33,16 @@ from std.benchmark import ( from std.benchmark.compiler import keep from emberregex import SetMatch, RegexSet from emberregex.set_bitnfa import ( - bitnfa_ex_idx_arr, - bitnfa_i32_arr, bitnfa_scan, - bitnfa_u64_arr, build_bitnfa, ) from emberregex.set_dfa import ( build_multi_dfa, - mdfa_pool_arr, mdfa_scan, mdfa_slices_arr, mdfa_table_str, ) -from emberregex.static_bytes import static_bytes +from emberregex.static_bytes import int_arr, list_arr, static_bytes # --------------------------------------------------------------------------- @@ -196,7 +192,7 @@ def mdfa_direct_scan[ comptime S = RegexSet[patterns] comptime MD = build_multi_dfa(S.nfa, S.nfa.can_use_dfa) comptime T = static_bytes[mdfa_table_str[MD.num_states * 256](MD)]() - comptime P = mdfa_pool_arr[len(MD.pool)](MD) + comptime P = int_arr[DType.int32, len(MD.pool)](MD.pool, 0) comptime SL = mdfa_slices_arr[6 * MD.num_states](MD) return mdfa_scan[d=MD, table=T, pool=P, slices=SL](input) @@ -550,11 +546,11 @@ def bench_set_bitnfa_log_dense(mut b: Bench) raises: # phase-3 acceptance comparison (target: within ~2x of the DFA). comptime S = RegexSet[LOG_PATS] comptime BN = build_bitnfa(S.nfa, True) - comptime REACH = bitnfa_u64_arr[256 * BN.lanes](BN.reach) - comptime EX = bitnfa_u64_arr[len(BN.ex_data)](BN.ex_data) - comptime EXIDX = bitnfa_ex_idx_arr[BN.num_positions](BN) - comptime POOL = bitnfa_i32_arr[len(BN.pool)](BN.pool) - comptime SLICES = bitnfa_i32_arr[12 * BN.num_positions](BN.slices) + comptime REACH = list_arr[UInt64, 256 * BN.lanes](BN.reach, 0) + comptime EX = list_arr[UInt64, len(BN.ex_data)](BN.ex_data, 0) + comptime EXIDX = int_arr[DType.int16, BN.num_positions](BN.ex_index, -1) + comptime POOL = int_arr[DType.int32, len(BN.pool)](BN.pool, 0) + comptime SLICES = int_arr[DType.int32, 12 * BN.num_positions](BN.slices, 0) var input = make_dense_haystack() @always_inline diff --git a/emberregex/engine.mojo b/emberregex/engine.mojo index 7bc885f..77ee782 100644 --- a/emberregex/engine.mojo +++ b/emberregex/engine.mojo @@ -43,8 +43,6 @@ from .optimize import ( extract_required_byte, extract_match_sandwich, is_pure_literal, - lit_bytes_arr, - lit_flags_arr, select_probe_offsets, FilterPrefix, InnerLiteral, @@ -87,7 +85,6 @@ from .static_dfa import ( _pivot_prefilter, build_eager_dfa, edfa_table_len, - edfa_flags_arr, edfa_id_dtype, edfa_full_match, edfa_match_at, @@ -101,9 +98,8 @@ from .static_lfdfa import ( from .static_rdfa import ( build_reverse_dfa, rdfa_find_start, - rdfa_flags_arr, ) -from .static_bytes import static_bytes, table_bytes +from .static_bytes import int_arr, list_arr, static_bytes, table_bytes from .sheng import ( sheng_cap_for, sheng_full_match, @@ -120,11 +116,10 @@ from .simd_kernels import ( ) from .executor import PikeVM, _VMBuffers, heapbt_match from .onepass import ( + ONEPASS_CLASS_LEN, OnePass, build_onepass, onepass_shape, - onepass_class_arr, - onepass_eps_arr, onepass_eps_len, onepass_match, onepass_state_arr, @@ -1356,7 +1351,9 @@ struct Regex[pattern: String](Copyable, Movable): comptime _EDFA_TABLE_S = table_bytes[Self._EDFA_DT]( Self._edfa.table, Self._EDFA_TN ) - comptime _EDFA_FLAGS = edfa_flags_arr[Self._edfa.num_states](Self._edfa) + comptime _EDFA_FLAGS = int_arr[DType.uint8, Self._edfa.num_states]( + Self._edfa.flags, 0 + ) comptime _EDFA_TABLE = static_bytes[Self._EDFA_TABLE_S]() # Narrowest tbl tier that holds this DFA: a 6-state DFA keeps 16-lane # masks even where 64 lanes are available (see sheng.mojo). @@ -1371,8 +1368,8 @@ struct Regex[pattern: String](Copyable, Movable): comptime _LFDFA_TABLE_S = table_bytes[Self._LFDFA_DT]( Self._lfdfa.d.table, Self._LFDFA_TN ) - comptime _LFDFA_FLAGS = edfa_flags_arr[Self._lfdfa.d.num_states]( - Self._lfdfa.d + comptime _LFDFA_FLAGS = int_arr[DType.uint8, Self._lfdfa.d.num_states]( + Self._lfdfa.d.flags, 0 ) comptime _LF_SHENG_CAP = sheng_cap_for(Self._lfdfa.d, Self._use_lf_sheng) comptime _LF_SHENG_MASKS_S = sheng_masks_str[Self._LF_SHENG_CAP]( @@ -1385,7 +1382,9 @@ struct Regex[pattern: String](Copyable, Movable): comptime _RDFA_TABLE_S = table_bytes[Self._RDFA_DT]( Self._rdfa.table, Self._RDFA_TN ) - comptime _RDFA_FLAGS = rdfa_flags_arr[Self._rdfa.num_states](Self._rdfa) + comptime _RDFA_FLAGS = int_arr[DType.uint8, Self._rdfa.num_states]( + Self._rdfa.flags, 0 + ) comptime _RDFA_TABLE = static_bytes[Self._RDFA_TABLE_S]() # Pivot-anchored prefilter shape (the `[class]+ P …` family), read off # the classic table; the leftmost-first scan starts at its candidate. @@ -1405,11 +1404,11 @@ struct Regex[pattern: String](Copyable, Movable): # _find_prefix_candidate's delegation to simd_find_literal_rare # (List-bearing values must not ride as comptime parameters). # Referenced only when fprefix_len >= 2. - comptime _FPRE_LIT = lit_bytes_arr[Self._strategy.fprefix_len]( - Self._fpre.bytes + comptime _FPRE_LIT = list_arr[UInt8, Self._strategy.fprefix_len]( + Self._fpre.bytes, 0 ) - comptime _FPRE_CL = lit_flags_arr[Self._strategy.fprefix_len]( - Self._fpre.caseless + comptime _FPRE_CL = list_arr[Bool, Self._strategy.fprefix_len]( + Self._fpre.caseless, False ) # Reverse-suffix / reverse-inner required literal (Rust regex's # ReverseSuffix/ReverseInner, effects (a)+(b) — see _lf_next_match's @@ -1433,8 +1432,10 @@ struct Regex[pattern: String](Copyable, Movable): # where _use_rev_literal holds, so the invalid case (length 0) never # elaborates. comptime _IL_N = len(Self._inner_lit.bytes) - comptime _IL_LIT = lit_bytes_arr[Self._IL_N](Self._inner_lit.bytes) - comptime _IL_CL = lit_flags_arr[Self._IL_N](Self._inner_lit.caseless) + comptime _IL_LIT = list_arr[UInt8, Self._IL_N](Self._inner_lit.bytes, 0) + comptime _IL_CL = list_arr[Bool, Self._IL_N]( + Self._inner_lit.caseless, False + ) comptime _IL_PROBES = select_probe_offsets( Self._inner_lit.bytes, Self._inner_lit.caseless ) @@ -1511,9 +1512,11 @@ struct Regex[pattern: String](Copyable, Movable): comptime _OP_TN = onepass_table_len(Self._onepass) comptime _OP_TABLE_S = onepass_table_str[Self._OP_TN](Self._onepass) comptime _OP_TABLE = static_bytes[Self._OP_TABLE_S]() - comptime _OP_CLASSES = onepass_class_arr(Self._onepass) + comptime _OP_CLASSES = int_arr[DType.uint8, ONEPASS_CLASS_LEN]( + Self._onepass.class_of, 0 + ) comptime _OP_NE = onepass_eps_len(Self._onepass) - comptime _OP_EPS = onepass_eps_arr[Self._OP_NE](Self._onepass) + comptime _OP_EPS = list_arr[UInt64, Self._OP_NE](Self._onepass.eps_sets, 0) comptime _OP_NS = onepass_state_len(Self._onepass) comptime _OP_STATES = onepass_state_arr[Self._OP_NS](Self._onepass) diff --git a/emberregex/onepass.mojo b/emberregex/onepass.mojo index 4f565c9..c9a4f2a 100644 --- a/emberregex/onepass.mojo +++ b/emberregex/onepass.mojo @@ -67,11 +67,11 @@ passes keeps the caller's -1. Tables (POD + Array rule: the struct crosses into the walkers as a comptime parameter, the bulk as separate arrays padded to at least EDFA_TABLE_MIN_BYTES so they lower to shared constant data): -`onepass_table_arr` — `num_states x nclasses` Int32 cells, -1 dead, else +`onepass_table_str` — `num_states x nclasses` Int32 cells, -1 dead, else the premultiplied next row, the next state id and the slot-set id packed -(`_OP_*` shifts); `onepass_class_arr` — byte to class; `onepass_eps_arr` — -slot bitsets by id (id 0 is the empty set); `onepass_state_arr` — per -state the match flags (`OP_*`) and the match slot-set id. +(`_OP_*` shifts); `class_of` — byte to class; `eps_sets` — slot bitsets +by id (id 0 is the empty set); `onepass_state_arr` — per state the match +flags (`OP_*`) and the match slot-set id. States that self-loop on all but a few bytes with no slot writes are accelerated exactly like the eager DFA's (`_edfa_accel_skip` over the @@ -703,14 +703,6 @@ def onepass_table_str[n: Int](op: OnePass) -> String: comptime ONEPASS_CLASS_LEN = EDFA_TABLE_MIN_BYTES -def onepass_class_arr(op: OnePass) -> Array[UInt8, ONEPASS_CLASS_LEN]: - """Comptime: byte -> class, padded (see ONEPASS_CLASS_LEN).""" - var arr = Array[UInt8, ONEPASS_CLASS_LEN](fill=0) - for b in range(256): - arr[b] = UInt8(op.class_of[b]) - return arr^ - - def onepass_eps_len(op: OnePass) -> Int: """Comptime: entry count of the slot-set array, padded to EDFA_TABLE_MIN_BYTES.""" @@ -719,17 +711,6 @@ def onepass_eps_len(op: OnePass) -> Int: return n if n > min_n else min_n -def onepass_eps_arr[n: Int](op: OnePass) -> Array[UInt64, n]: - """Comptime: slot bitset per slot-set id.""" - var arr = Array[UInt64, n](fill=0) - var m = len(op.eps_sets) - if n < m: - m = n - for i in range(m): - arr[i] = op.eps_sets[i] - return arr^ - - def onepass_state_len(op: OnePass) -> Int: """Comptime: entry count of the per-state array, padded to EDFA_TABLE_MIN_BYTES.""" diff --git a/emberregex/optimize.mojo b/emberregex/optimize.mojo index d973342..2ab71d9 100644 --- a/emberregex/optimize.mojo +++ b/emberregex/optimize.mojo @@ -1005,23 +1005,6 @@ def extract_inner_literal(nfa: NFA, cyclic: List[Bool]) -> InnerLiteral: return res^ -def lit_bytes_arr[n: Int](l: List[UInt8]) -> Array[UInt8, n]: - """Comptime: List -> Array so literal bytes can ride as walker - comptime parameters (List-bearing values must not).""" - var a = Array[UInt8, n](fill=0) - for i in range(min(n, len(l))): - a[i] = l[i] - return a^ - - -def lit_flags_arr[n: Int](l: List[Bool]) -> Array[Bool, n]: - """Comptime: List -> Array for the parallel caseless flags.""" - var a = Array[Bool, n](fill=False) - for i in range(min(n, len(l))): - a[i] = l[i] - return a^ - - def _probe_rank_vec() -> SIMD[DType.int32, 256]: """Comptime: approximate background byte frequency (0 = rarest, 255 = most common) over typical text/code, for prefilter probe selection. diff --git a/emberregex/set_ac.mojo b/emberregex/set_ac.mojo index 1e54d68..1c8e0e3 100644 --- a/emberregex/set_ac.mojo +++ b/emberregex/set_ac.mojo @@ -508,13 +508,6 @@ def ac_rep_arr[n: Int](d: ACSet) -> Array[Int32, n]: return arr^ -def ac_pool_arr[n: Int](d: ACSet) -> Array[Int32, n]: - var arr = Array[Int32, n](fill=0) - for i in range(n): - arr[i] = Int32(d.pool[i]) - return arr^ - - # --- Runtime walker --------------------------------------------------------- diff --git a/emberregex/set_bitnfa.mojo b/emberregex/set_bitnfa.mojo index 5fb61dd..9d4db3e 100644 --- a/emberregex/set_bitnfa.mojo +++ b/emberregex/set_bitnfa.mojo @@ -440,27 +440,6 @@ def build_bitnfa(nfa: NFA, enabled: Bool) -> BitNFA: # --- Comptime materialization helpers --------------------------------------- -def bitnfa_u64_arr[n: Int](data: List[UInt64]) -> Array[UInt64, n]: - var arr = Array[UInt64, n](fill=0) - for i in range(n): - arr[i] = data[i] - return arr^ - - -def bitnfa_i32_arr[n: Int](data: List[Int]) -> Array[Int32, n]: - var arr = Array[Int32, n](fill=0) - for i in range(n): - arr[i] = Int32(data[i]) - return arr^ - - -def bitnfa_ex_idx_arr[n: Int](d: BitNFA) -> Array[Int16, n]: - var arr = Array[Int16, n](fill=-1) - for i in range(n): - arr[i] = Int16(d.ex_index[i]) - return arr^ - - # --- Runtime walker ---------------------------------------------------------- diff --git a/emberregex/set_combine.mojo b/emberregex/set_combine.mojo index a7828a0..28f4668 100644 --- a/emberregex/set_combine.mojo +++ b/emberregex/set_combine.mojo @@ -196,13 +196,6 @@ def _eval[ return stack[0] -def combos_rpn_arr[n: Int](pool: List[Int]) -> Array[Int32, n]: - var arr = Array[Int32, n](fill=0) - for i in range(min(n, len(pool))): - arr[i] = Int32(pool[i]) - return arr^ - - def evaluate_combinations[ n: Int, //, rpn: Array[Int32, n], num_combos: Int, num_patterns: Int ](reports: List[SetMatch]) -> List[SetMatch]: diff --git a/emberregex/set_dfa.mojo b/emberregex/set_dfa.mojo index 5e7cd6e..ac392ba 100644 --- a/emberregex/set_dfa.mojo +++ b/emberregex/set_dfa.mojo @@ -718,13 +718,6 @@ def mdfa_table_str[n: Int](d: MultiDFA) -> String: return table_bytes[DType.int16](d.table, n) -def mdfa_pool_arr[n: Int](d: MultiDFA) -> Array[Int32, n]: - var arr = Array[Int32, n](fill=0) - for i in range(n): - arr[i] = Int32(d.pool[i]) - return arr^ - - def mdfa_slices_arr[n: Int](d: MultiDFA) -> Array[Int32, n]: """Per-state slice metadata, interleaved as 6 Int32 per state: (norm_off, norm_len, nl_off, nl_len, end_off, end_len).""" diff --git a/emberregex/set_engine.mojo b/emberregex/set_engine.mojo index 5f4683c..f1be3ee 100644 --- a/emberregex/set_engine.mojo +++ b/emberregex/set_engine.mojo @@ -49,10 +49,9 @@ from std.math import max from std.os import abort from .nfa import NFA -from .static_bytes import static_bytes +from .static_bytes import int_arr, list_arr, static_bytes from .set_ac import ( ac_cls_arr, - ac_pool_arr, ac_rep_arr, ac_scan, ac_table_str, @@ -60,15 +59,11 @@ from .set_ac import ( build_ac, ) from .set_bitnfa import ( - bitnfa_ex_idx_arr, - bitnfa_i32_arr, bitnfa_scan, - bitnfa_u64_arr, build_bitnfa, ) from .set_dfa import ( build_multi_dfa, - mdfa_pool_arr, mdfa_scan, mdfa_slices_arr, mdfa_table_str, @@ -87,12 +82,12 @@ from .set_rose import ( merge_reports, rose_flags_arr, rose_bcls_arr, + _rose_lits, + _rose_meta, rose_bcls_len, - rose_lits_arr, rose_lits_len, rose_look_arr, rose_look_len, - rose_meta_arr, rose_meta_len, rose_scan, rose_table_str, @@ -101,7 +96,6 @@ from .set_rose import ( from .set_reverse import ( build_reverse_dfa, leftmost_nonoverlapping, - rdfa_pool_arr, rdfa_slices_arr, rdfa_table_str, rdfa_view, @@ -110,7 +104,6 @@ from .set_reverse import ( from .set_combine import ( combos_error, combos_rpn, - combos_rpn_arr, evaluate_combinations, ) from .set_prefilter import confirm_span @@ -259,7 +252,9 @@ struct RegexSet[ ](Self._ac) comptime _AC_CLS = ac_cls_arr(Self._ac) comptime _AC_REP = ac_rep_arr[2 * Self._ac.num_states](Self._ac) - comptime _AC_POOL = ac_pool_arr[len(Self._ac.pool)](Self._ac) + comptime _AC_POOL = int_arr[DType.int32, len(Self._ac.pool)]( + Self._ac.pool, 0 + ) # --- Rose lane: literal decomposition (phase 4) ------------------------- # Extraction is linear, so this decides before anything determinizes. @@ -281,8 +276,12 @@ struct RegexSet[ # mangled into symbol names, and carrying the confirm table there too # blew the linker's symbol-length limit on a 32-pattern set. comptime _rose_v = rose_view(Self._rose) - comptime _ROSE_META = rose_meta_arr[rose_meta_len(Self._rose)](Self._rose) - comptime _ROSE_LITS = rose_lits_arr[rose_lits_len(Self._rose)](Self._rose) + comptime _ROSE_META = int_arr[DType.int32, rose_meta_len(Self._rose)]( + _rose_meta(Self._rose), 0 + ) + comptime _ROSE_LITS = int_arr[DType.int32, rose_lits_len(Self._rose)]( + _rose_lits(Self._rose), 0 + ) comptime _ROSE_BCLS = rose_bcls_arr[rose_bcls_len(Self._rose)](Self._rose) comptime _ROSE_LOOK = rose_look_arr[rose_look_len(Self._rose)](Self._rose) @@ -304,7 +303,9 @@ struct RegexSet[ comptime _RES_TABLE_S = mdfa_table_str[Self._res_mdfa.num_states * 256]( Self._res_mdfa ) - comptime _RES_POOL = mdfa_pool_arr[len(Self._res_mdfa.pool)](Self._res_mdfa) + comptime _RES_POOL = int_arr[DType.int32, len(Self._res_mdfa.pool)]( + Self._res_mdfa.pool, 0 + ) comptime _RES_SLICES = mdfa_slices_arr[6 * Self._res_mdfa.num_states]( Self._res_mdfa ) @@ -315,21 +316,21 @@ struct RegexSet[ and not Self._use_res_mdfa, ) comptime _use_res_bitnfa = Self._res_bitnfa.valid - comptime _RES_BN_REACH = bitnfa_u64_arr[256 * Self._res_bitnfa.lanes]( - Self._res_bitnfa.reach + comptime _RES_BN_REACH = list_arr[UInt64, 256 * Self._res_bitnfa.lanes]( + Self._res_bitnfa.reach, 0 ) - comptime _RES_BN_EX = bitnfa_u64_arr[len(Self._res_bitnfa.ex_data)]( - Self._res_bitnfa.ex_data + comptime _RES_BN_EX = list_arr[UInt64, len(Self._res_bitnfa.ex_data)]( + Self._res_bitnfa.ex_data, 0 ) - comptime _RES_BN_EXIDX = bitnfa_ex_idx_arr[Self._res_bitnfa.num_positions]( - Self._res_bitnfa + comptime _RES_BN_EXIDX = int_arr[ + DType.int16, Self._res_bitnfa.num_positions + ](Self._res_bitnfa.ex_index, -1) + comptime _RES_BN_POOL = int_arr[DType.int32, len(Self._res_bitnfa.pool)]( + Self._res_bitnfa.pool, 0 ) - comptime _RES_BN_POOL = bitnfa_i32_arr[len(Self._res_bitnfa.pool)]( - Self._res_bitnfa.pool - ) - comptime _RES_BN_SLICES = bitnfa_i32_arr[ - 12 * Self._res_bitnfa.num_positions - ](Self._res_bitnfa.slices) + comptime _RES_BN_SLICES = int_arr[ + DType.int32, 12 * Self._res_bitnfa.num_positions + ](Self._res_bitnfa.slices, 0) comptime _use_res_pike = ( Self._has_residual and not Self._use_res_mdfa @@ -351,7 +352,9 @@ struct RegexSet[ comptime _MDFA_TABLE_S = mdfa_table_str[Self._mdfa.num_states * 256]( Self._mdfa ) - comptime _MDFA_POOL = mdfa_pool_arr[len(Self._mdfa.pool)](Self._mdfa) + comptime _MDFA_POOL = int_arr[DType.int32, len(Self._mdfa.pool)]( + Self._mdfa.pool, 0 + ) comptime _MDFA_SLICES = mdfa_slices_arr[6 * Self._mdfa.num_states]( Self._mdfa ) @@ -379,20 +382,20 @@ struct RegexSet[ and not Self._use_mdfa and Self._bitnfa.valid ) - comptime _BN_REACH = bitnfa_u64_arr[256 * Self._bitnfa.lanes]( - Self._bitnfa.reach + comptime _BN_REACH = list_arr[UInt64, 256 * Self._bitnfa.lanes]( + Self._bitnfa.reach, 0 ) - comptime _BN_EX = bitnfa_u64_arr[len(Self._bitnfa.ex_data)]( - Self._bitnfa.ex_data + comptime _BN_EX = list_arr[UInt64, len(Self._bitnfa.ex_data)]( + Self._bitnfa.ex_data, 0 ) - comptime _BN_EXIDX = bitnfa_ex_idx_arr[Self._bitnfa.num_positions]( - Self._bitnfa + comptime _BN_EXIDX = int_arr[DType.int16, Self._bitnfa.num_positions]( + Self._bitnfa.ex_index, -1 ) - comptime _BN_POOL = bitnfa_i32_arr[len(Self._bitnfa.pool)]( - Self._bitnfa.pool + comptime _BN_POOL = int_arr[DType.int32, len(Self._bitnfa.pool)]( + Self._bitnfa.pool, 0 ) - comptime _BN_SLICES = bitnfa_i32_arr[12 * Self._bitnfa.num_positions]( - Self._bitnfa.slices + comptime _BN_SLICES = int_arr[DType.int32, 12 * Self._bitnfa.num_positions]( + Self._bitnfa.slices, 0 ) # Stream aliases: same arrays, named for the streaming API so # set_stream.mojo does not reach into block-lane internals. @@ -429,7 +432,9 @@ struct RegexSet[ comptime _RD_TABLE_S = rdfa_table_str[Self._rdfa.num_states * 256]( Self._rdfa ) - comptime _RD_POOL = rdfa_pool_arr[len(Self._rdfa.pool)](Self._rdfa) + comptime _RD_POOL = int_arr[DType.int32, len(Self._rdfa.pool)]( + Self._rdfa.pool, 0 + ) comptime _RD_SLICES = rdfa_slices_arr[6 * Self._rdfa.num_states](Self._rdfa) # --- Exact backrefs / lookaround (phase 7) ----------------------------- @@ -441,9 +446,9 @@ struct RegexSet[ # --- Logical combinations (phase 7) ------------------------------------ comptime _num_combos = len(Self.combos) comptime _combos_ok = _check_combos(Self.combos, Self.num_patterns) - comptime _COMBO_RPN = combos_rpn_arr[ - max(1, len(combos_rpn(Self.combos, Self.num_patterns))) - ](combos_rpn(Self.combos, Self.num_patterns)) + comptime _COMBO_RPN = int_arr[ + DType.int32, max(1, len(combos_rpn(Self.combos, Self.num_patterns))) + ](combos_rpn(Self.combos, Self.num_patterns), 0) # --- Semantic surface (phase 7) ---------------------------------------- comptime _has_sem = has_semantics(Self.flags, Self.ext, Self.num_patterns) diff --git a/emberregex/set_reverse.mojo b/emberregex/set_reverse.mojo index b9c151b..67bf96a 100644 --- a/emberregex/set_reverse.mojo +++ b/emberregex/set_reverse.mojo @@ -835,13 +835,6 @@ def rdfa_table_str[n: Int](d: ReverseDFA) -> String: return table_bytes[DType.int32](d.table, n) -def rdfa_pool_arr[n: Int](d: ReverseDFA) -> Array[Int32, n]: - var arr = Array[Int32, n](fill=0) - for i in range(n): - arr[i] = Int32(d.pool[i]) - return arr^ - - def rdfa_slices_arr[n: Int](d: ReverseDFA) -> Array[Int32, n]: """Per-state slice metadata, 6 Int32 per state: (norm_off, norm_len, bol0_off, bol0_len, bolnl_off, bolnl_len).""" diff --git a/emberregex/set_rose.mojo b/emberregex/set_rose.mojo index da2e350..1940028 100644 --- a/emberregex/set_rose.mojo +++ b/emberregex/set_rose.mojo @@ -257,7 +257,7 @@ struct RoseView(Copyable, Movable): - the same data as `Array` costs ~4 chars per element. So the per-entry pools travel as Array parameters - (rose_meta_arr / rose_lits_arr) and this struct carries only scalars. + (`_rose_meta` / `_rose_lits`) and this struct carries only scalars. The other lanes never hit this because their scan functions stay small enough to inline, at which point no symbol spells the values at all. @@ -344,14 +344,6 @@ def rose_lits_len(r: RoseSet) -> Int: return max(1, n) -def rose_meta_arr[n: Int](r: RoseSet) -> Array[Int32, n]: - var arr = Array[Int32, n](fill=0) - var meta = _rose_meta(r) - for i in range(min(n, len(meta))): - arr[i] = Int32(meta[i]) - return arr^ - - def rose_bcls_len(r: RoseSet) -> Int: return max(1, 8 * len(r.back_classes)) @@ -390,14 +382,6 @@ def rose_look_arr[n: Int](r: RoseSet) -> Array[Int32, n]: return arr^ -def rose_lits_arr[n: Int](r: RoseSet) -> Array[Int32, n]: - var arr = Array[Int32, n](fill=0) - var lits = _rose_lits(r) - for i in range(min(n, len(lits))): - arr[i] = Int32(lits[i]) - return arr^ - - def _entry_bytes[ mn: Int, ln: Int ](meta: Array[Int32, mn], lits: Array[Int32, ln], i: Int) -> List[Int]: diff --git a/emberregex/static_bytes.mojo b/emberregex/static_bytes.mojo index 5256a93..6185ff7 100644 --- a/emberregex/static_bytes.mojo +++ b/emberregex/static_bytes.mojo @@ -12,7 +12,7 @@ bytes at comptime (`table_bytes`), turned into a `StringLiteral` once per `Regex` (`static_bytes`), and read through `unsafe_ptr()` in the walkers. """ -from std.collections import List +from std.collections import Array, List from std.collections.string.string_span import _get_kgen_string from std.sys import size_of @@ -68,3 +68,26 @@ def static_bytes[ ]: """Comptime: the bytes of `s` as a string literal (static data).""" return {} + + +def list_arr[ + T: Copyable & Deinitable, n: Int +](l: List[T], fill: T) -> Array[T, n]: + """Comptime: the first `n` entries of `l` as an Array, `fill` past its + end — the form small tables take to ride as walker comptime parameters + (List-bearing values must not). Call at decl level (`comptime x = + list_arr[...](...)`) so the copy is memoized.""" + var a = Array[T, n](fill=fill) + for i in range(min(n, len(l))): + a[i] = l[i].copy() + return a^ + + +def int_arr[ + dt: DType, n: Int +](l: List[Int], fill: Scalar[dt]) -> Array[Scalar[dt], n]: + """Comptime: `list_arr` narrowing each entry to `dt`.""" + var a = Array[Scalar[dt], n](fill=fill) + for i in range(min(n, len(l))): + a[i] = Scalar[dt](l[i]) + return a^ diff --git a/emberregex/static_dfa.mojo b/emberregex/static_dfa.mojo index b2969e5..ceced9c 100644 --- a/emberregex/static_dfa.mojo +++ b/emberregex/static_dfa.mojo @@ -20,6 +20,7 @@ from .ast import AnchorKind from .constants import CHAR_NEWLINE, is_word_byte from .nfa import NFA, NFAStateKind from .optimize import PROBE_RANKS +from .static_bytes import int_arr from .dfa import _reaches_match from .charset import BITMAP_WIDTH from .simd_scan import first_lane_index, lane_bits, simd_find_byte @@ -548,7 +549,8 @@ struct EagerDFA(Copyable, Movable): """Comptime-computed DFA: flat transition table + per-state flags. Only ever exists as a comptime value; the runtime engine reads the - materialized Array forms (see edfa_table_arr / edfa_flags_arr). + materialized forms (the table as a string literal, the flags as an + Array — `Regex._EDFA_TABLE` / `_EDFA_FLAGS`). """ var valid: Bool @@ -1840,14 +1842,6 @@ def _edfa_finish( return pstarts^ -def edfa_flags_arr[n: Int](d: EagerDFA) -> Array[UInt8, n]: - """Comptime conversion of per-state flags to a materializable array.""" - var arr = Array[UInt8, n](fill=0) - for i in range(n): - arr[i] = UInt8(d.flags[i]) - return arr^ - - # --- Runtime table walkers ------------------------------------------------- # # The DFA metadata `d` and the table/flags arrive as comptime parameters, so @@ -1892,14 +1886,6 @@ def _accel_mask_word(d: EagerDFA, word: Int) -> UInt64: return m -def _region_land_arr(d: EagerDFA) -> Array[Int16, 256]: - """Comptime: `region_land` as a materializable array.""" - var arr = Array[Int16, 256](fill=-1) - for b in range(len(d.region_land)): - arr[b] = Int16(d.region_land[b]) - return arr^ - - @always_inline def _edfa_accel_skip[ origin: Origin, //, d: EagerDFA @@ -2005,7 +1991,7 @@ def _edfa_region_skip[ if p2 > p: # Every skipped byte's target is the same from any # member: the state is whatever the last one selected. - comptime land = _region_land_arr(d) + comptime land = int_arr[DType.int16, 256](d.region_land, -1) var lnd = materialize[land]() cur = Int(lnd.unsafe_get(Int(input.unsafe_get(p2 - 1)))) p = p2 diff --git a/emberregex/static_rdfa.mojo b/emberregex/static_rdfa.mojo index 56b1daa..8031979 100644 --- a/emberregex/static_rdfa.mojo +++ b/emberregex/static_rdfa.mojo @@ -743,13 +743,6 @@ def build_reverse_dfa(nfa: NFA, enabled: Bool) -> RDFA: return result^ -def rdfa_flags_arr[n: Int](d: RDFA) -> Array[UInt8, n]: - var arr = Array[UInt8, n](fill=0) - for i in range(n): - arr[i] = UInt8(d.flags[i]) - return arr^ - - @always_inline def _rfind_exit2[ origin: Origin, //, e1: UInt8, e2: UInt8 diff --git a/test/test_leftmost_first_dfa.mojo b/test/test_leftmost_first_dfa.mojo index e398519..3be6c8e 100644 --- a/test/test_leftmost_first_dfa.mojo +++ b/test/test_leftmost_first_dfa.mojo @@ -13,11 +13,10 @@ newlines and bytes >= 0x80. from emberregex import Regex from emberregex.simd_kernels import HAS_FAST_BYTE_SHUFFLE -from emberregex.static_bytes import static_bytes, table_bytes +from emberregex.static_bytes import int_arr, static_bytes, table_bytes from emberregex.static_dfa import ( EDFA_TABLE_MIN_BYTES, EagerDFA, - edfa_flags_arr, edfa_id_dtype, ) from emberregex.static_rdfa import rdfa_find_start @@ -690,7 +689,7 @@ def test_wide_list_signature_renumbering() raises: comptime tn = lf.d.num_states * 256 comptime dt = edfa_id_dtype(lf.d.num_states) comptime table = static_bytes[table_bytes[dt](lf.d.table, tn)]() - comptime flags = edfa_flags_arr[lf.d.num_states](lf.d) + comptime flags = int_arr[DType.uint8, lf.d.num_states](lf.d.flags, 0) var input = String("!!!!!!!!!!!!!!!!!!!!q!!") var bytes = input.as_bytes() assert_equal(lfdfa_find_end[lf=lf, table=table, flags=flags](bytes, 0), 21) diff --git a/test/test_onepass.mojo b/test/test_onepass.mojo index dde51b9..9e2de7c 100644 --- a/test/test_onepass.mojo +++ b/test/test_onepass.mojo @@ -19,8 +19,9 @@ assertion so a selection change cannot silently move a test onto — or off """ from emberregex import Regex -from emberregex.static_bytes import static_bytes +from emberregex.static_bytes import int_arr, list_arr, static_bytes from emberregex.onepass import ( + ONEPASS_CLASS_LEN, OnePass, OP_MATCH, OP_NEED_EOL, @@ -28,8 +29,6 @@ from emberregex.onepass import ( OP_NEED_WORD, ONEPASS_STATE_CAP, build_onepass, - onepass_class_arr, - onepass_eps_arr, onepass_eps_len, onepass_match, onepass_state_arr, @@ -201,9 +200,9 @@ def test_onepass_walker_acceleration() raises: assert_true(accel >= 1) comptime TN = onepass_table_len(op) comptime TBL = static_bytes[onepass_table_str[TN](op)]() - comptime CLS = onepass_class_arr(op) + comptime CLS = int_arr[DType.uint8, ONEPASS_CLASS_LEN](op.class_of, 0) comptime NE = onepass_eps_len(op) - comptime EPS = onepass_eps_arr[NE](op) + comptime EPS = list_arr[UInt64, NE](op.eps_sets, 0) comptime NS = onepass_state_len(op) comptime ST = onepass_state_arr[NS](op) # 40-byte middle run so the 16-byte-vector acceleration fires. diff --git a/test/test_reverse_literal.mojo b/test/test_reverse_literal.mojo index 55ba4db..b72c23f 100644 --- a/test/test_reverse_literal.mojo +++ b/test/test_reverse_literal.mojo @@ -19,7 +19,7 @@ matches, adjacent matches, matches at 0 and EOF, newlines and bytes >= """ from emberregex import Regex -from emberregex.optimize import lit_bytes_arr, lit_flags_arr +from emberregex.static_bytes import list_arr from emberregex.simd_scan import simd_find_literal_rare from std.benchmark import keep from std.testing import assert_true, assert_false, assert_equal, TestSuite @@ -211,10 +211,10 @@ def test_strategy_off_without_a_literal_or_off_lane() raises: # --- The memmem kernel ------------------------------------------------------- -comptime _TXT_LIT = lit_bytes_arr[4]([0x2E, 0x74, 0x78, 0x74]) # ".txt" -comptime _TXT_CL = lit_flags_arr[4]([False, False, False, False]) -comptime _AB_LIT = lit_bytes_arr[2]([0x61, 0x62]) # "ab" -comptime _AB_CL_A = lit_flags_arr[2]([True, False]) # caseless 'a' +comptime _TXT_LIT = list_arr[UInt8, 4]([0x2E, 0x74, 0x78, 0x74], 0) # ".txt" +comptime _TXT_CL = list_arr[Bool, 4]([False, False, False, False], False) +comptime _AB_LIT = list_arr[UInt8, 2]([0x61, 0x62], 0) # "ab" +comptime _AB_CL_A = list_arr[Bool, 2]([True, False], False) # caseless 'a' def _find_txt(input: String, start: Int) -> Int: diff --git a/test/test_set_ac.mojo b/test/test_set_ac.mojo index 7cbdee1..d490f82 100644 --- a/test/test_set_ac.mojo +++ b/test/test_set_ac.mojo @@ -18,13 +18,12 @@ tests one rung down the ladder: """ from emberregex import SetMatch, SetSpan, RegexSet -from emberregex.static_bytes import static_bytes +from emberregex.static_bytes import int_arr, static_bytes from emberregex.set_ac import ( AC_MAX, AC_POOL_CAP, AC_TABLE_CAP, ac_cls_arr, - ac_pool_arr, ac_rep_arr, ac_scan, ac_table_str, @@ -70,7 +69,7 @@ def ac_direct_scan[ comptime T = static_bytes[ac_table_str[A.num_states * A.num_classes](A)]() comptime C = ac_cls_arr(A) comptime R = ac_rep_arr[2 * A.num_states](A) - comptime P = ac_pool_arr[len(A.pool)](A) + comptime P = int_arr[DType.int32, len(A.pool)](A.pool, 0) return ac_scan[v=V, table=T, cls=C, rep=R, pool=P](input) diff --git a/test/test_set_bench_coverage.mojo b/test/test_set_bench_coverage.mojo index bbbf83c..663e626 100644 --- a/test/test_set_bench_coverage.mojo +++ b/test/test_set_bench_coverage.mojo @@ -8,16 +8,12 @@ Keep the corpora builders in sync with bench_set.mojo. from emberregex import SetMatch, RegexSet from emberregex.set_bitnfa import ( - bitnfa_ex_idx_arr, - bitnfa_i32_arr, bitnfa_scan, - bitnfa_u64_arr, build_bitnfa, ) -from emberregex.static_bytes import static_bytes +from emberregex.static_bytes import int_arr, list_arr, static_bytes from emberregex.set_dfa import ( build_multi_dfa, - mdfa_pool_arr, mdfa_scan, mdfa_slices_arr, mdfa_table_str, @@ -144,7 +140,7 @@ def mdfa_direct_scan[ comptime S = RegexSet[patterns] comptime MD = build_multi_dfa(S.nfa, S.nfa.can_use_dfa) comptime T = static_bytes[mdfa_table_str[MD.num_states * 256](MD)]() - comptime P = mdfa_pool_arr[len(MD.pool)](MD) + comptime P = int_arr[DType.int32, len(MD.pool)](MD.pool, 0) comptime SL = mdfa_slices_arr[6 * MD.num_states](MD) return mdfa_scan[d=MD, table=T, pool=P, slices=SL](input) @@ -448,11 +444,11 @@ def test_bench_bitnfa_log_direct() raises: comptime BN = build_bitnfa(S.nfa, True) comptime bn_valid = BN.valid assert_true(bn_valid) - comptime REACH = bitnfa_u64_arr[256 * BN.lanes](BN.reach) - comptime EX = bitnfa_u64_arr[len(BN.ex_data)](BN.ex_data) - comptime EXIDX = bitnfa_ex_idx_arr[BN.num_positions](BN) - comptime POOL = bitnfa_i32_arr[len(BN.pool)](BN.pool) - comptime SLICES = bitnfa_i32_arr[12 * BN.num_positions](BN.slices) + comptime REACH = list_arr[UInt64, 256 * BN.lanes](BN.reach, 0) + comptime EX = list_arr[UInt64, len(BN.ex_data)](BN.ex_data, 0) + comptime EXIDX = int_arr[DType.int16, BN.num_positions](BN.ex_index, -1) + comptime POOL = int_arr[DType.int32, len(BN.pool)](BN.pool, 0) + comptime SLICES = int_arr[DType.int32, 12 * BN.num_positions](BN.slices, 0) var input = make_dense_haystack() var got = bitnfa_scan[ d=BN, diff --git a/test/test_set_paths.mojo b/test/test_set_paths.mojo index c3ce8db..e4372c0 100644 --- a/test/test_set_paths.mojo +++ b/test/test_set_paths.mojo @@ -18,11 +18,9 @@ that must treat SAVE as epsilon and BACKREF as unrunnable get pinned too. from emberregex import RegexSet from emberregex.nfa import build_nfa, NFA, NFAStateKind from emberregex.parser import parse +from emberregex.static_bytes import int_arr, list_arr from emberregex.set_bitnfa import ( - bitnfa_ex_idx_arr, - bitnfa_i32_arr, bitnfa_scan, - bitnfa_u64_arr, build_bitnfa, BITNFA_POS_CAP, ) @@ -114,11 +112,11 @@ def _bitnfa_scan_direct[ """Scan on the bit-parallel NFA, bypassing engine selection.""" comptime S = RegexSet[patterns] comptime BN = build_bitnfa(S.nfa, S.nfa.can_use_dfa) - comptime REACH = bitnfa_u64_arr[256 * BN.lanes](BN.reach) - comptime EX = bitnfa_u64_arr[len(BN.ex_data)](BN.ex_data) - comptime EXIDX = bitnfa_ex_idx_arr[BN.num_positions](BN) - comptime POOL = bitnfa_i32_arr[len(BN.pool)](BN.pool) - comptime SLICES = bitnfa_i32_arr[12 * BN.num_positions](BN.slices) + comptime REACH = list_arr[UInt64, 256 * BN.lanes](BN.reach, 0) + comptime EX = list_arr[UInt64, len(BN.ex_data)](BN.ex_data, 0) + comptime EXIDX = int_arr[DType.int16, BN.num_positions](BN.ex_index, -1) + comptime POOL = int_arr[DType.int32, len(BN.pool)](BN.pool, 0) + comptime SLICES = int_arr[DType.int32, 12 * BN.num_positions](BN.slices, 0) return bitnfa_scan[ d=BN, reach=REACH, diff --git a/test/test_set_phase2.mojo b/test/test_set_phase2.mojo index 0d279c6..971bf31 100644 --- a/test/test_set_phase2.mojo +++ b/test/test_set_phase2.mojo @@ -13,10 +13,9 @@ instead, which would silently stop exercising this engine. """ from emberregex import SetMatch, RegexSet -from emberregex.static_bytes import static_bytes +from emberregex.static_bytes import int_arr, static_bytes from emberregex.set_dfa import ( build_multi_dfa, - mdfa_pool_arr, mdfa_scan, mdfa_slices_arr, mdfa_table_str, @@ -35,7 +34,7 @@ def _mdfa_scan[ comptime S = RegexSet[patterns] comptime MD = build_multi_dfa(S.nfa, S.nfa.can_use_dfa) comptime T = static_bytes[mdfa_table_str[MD.num_states * 256](MD)]() - comptime P = mdfa_pool_arr[len(MD.pool)](MD) + comptime P = int_arr[DType.int32, len(MD.pool)](MD.pool, 0) comptime SL = mdfa_slices_arr[6 * MD.num_states](MD) return mdfa_scan[d=MD, table=T, pool=P, slices=SL](input) diff --git a/test/test_set_phase3.mojo b/test/test_set_phase3.mojo index 074e4af..ab4d76d 100644 --- a/test/test_set_phase3.mojo +++ b/test/test_set_phase3.mojo @@ -14,11 +14,9 @@ keep their `_use_bitnfa` pins. """ from emberregex import SetMatch, RegexSet +from emberregex.static_bytes import int_arr, list_arr from emberregex.set_bitnfa import ( - bitnfa_ex_idx_arr, - bitnfa_i32_arr, bitnfa_scan, - bitnfa_u64_arr, build_bitnfa, ) from emberregex.set_nfa import build_union_nfa @@ -32,11 +30,11 @@ def _bitnfa_scan_direct[ """Scan on the bit-parallel NFA, bypassing engine selection.""" comptime S = RegexSet[patterns] comptime BN = build_bitnfa(S.nfa, S.nfa.can_use_dfa) - comptime REACH = bitnfa_u64_arr[256 * BN.lanes](BN.reach) - comptime EX = bitnfa_u64_arr[len(BN.ex_data)](BN.ex_data) - comptime EXIDX = bitnfa_ex_idx_arr[BN.num_positions](BN) - comptime POOL = bitnfa_i32_arr[len(BN.pool)](BN.pool) - comptime SLICES = bitnfa_i32_arr[12 * BN.num_positions](BN.slices) + comptime REACH = list_arr[UInt64, 256 * BN.lanes](BN.reach, 0) + comptime EX = list_arr[UInt64, len(BN.ex_data)](BN.ex_data, 0) + comptime EXIDX = int_arr[DType.int16, BN.num_positions](BN.ex_index, -1) + comptime POOL = int_arr[DType.int32, len(BN.pool)](BN.pool, 0) + comptime SLICES = int_arr[DType.int32, 12 * BN.num_positions](BN.slices, 0) return bitnfa_scan[ d=BN, reach=REACH, diff --git a/test/test_word_boundary_dfa.mojo b/test/test_word_boundary_dfa.mojo index 2e25618..1c99c59 100644 --- a/test/test_word_boundary_dfa.mojo +++ b/test/test_word_boundary_dfa.mojo @@ -13,7 +13,7 @@ shapes engine selection leaves on the backtracker. """ from emberregex import Regex -from emberregex.static_bytes import static_bytes, table_bytes +from emberregex.static_bytes import int_arr, static_bytes, table_bytes from emberregex.static_dfa import ( EDFA_MATCH_IF_NONWORD, EDFA_MATCH_IF_WORD, @@ -21,7 +21,6 @@ from emberregex.static_dfa import ( _edfa_has_region, _is_word_byte, build_eager_dfa, - edfa_flags_arr, edfa_full_match, edfa_id_dtype, edfa_match_at, @@ -30,7 +29,6 @@ from emberregex.static_lfdfa import LFDFA, build_lf_dfa, lfdfa_find_end from emberregex.static_rdfa import ( build_reverse_dfa, rdfa_find_start, - rdfa_flags_arr, ) from std.testing import assert_true, assert_false, assert_equal, TestSuite @@ -238,19 +236,19 @@ def _forced_lane_check[p: StaticString](input: String, label: String) raises: comptime ETN = ed.num_states * 256 comptime EDT = edfa_id_dtype(ed.num_states) comptime etbl = static_bytes[table_bytes[EDT](ed.table, ETN)]() - comptime efl = edfa_flags_arr[ed.num_states](ed) + comptime efl = int_arr[DType.uint8, ed.num_states](ed.flags, 0) comptime lf = build_lf_dfa(nfa, True) comptime assert lf.valid comptime LTN = lf.d.num_states * 256 comptime LDT = edfa_id_dtype(lf.d.num_states) comptime ltbl = static_bytes[table_bytes[LDT](lf.d.table, LTN)]() - comptime lfl = edfa_flags_arr[lf.d.num_states](lf.d) + comptime lfl = int_arr[DType.uint8, lf.d.num_states](lf.d.flags, 0) comptime rd = build_reverse_dfa(nfa, True) comptime assert rd.valid comptime RTN = rd.num_states * 256 comptime RDT = edfa_id_dtype(rd.num_states) comptime rtbl = static_bytes[table_bytes[RDT](rd.table, RTN)]() - comptime rfl = rdfa_flags_arr[rd.num_states](rd) + comptime rfl = int_arr[DType.uint8, rd.num_states](rd.flags, 0) var re = Regex[p]() var bytes = input.as_bytes() From e28a4da27cec01a594a46b6313cf578fbb471c2e Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:12:50 -0600 Subject: [PATCH 40/65] cleanup: fold teddy_find_prefix into teddy_search_forward[want_end=False]; import _NibbleTable The prefilter was search_forward's SIMD candidate loop and tail with a first-hit verify instead of the longest-end one; want_end=False keeps that verify, so the per-candidate work is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/engine.mojo | 5 ++- emberregex/teddy.mojo | 92 +++++++++++++----------------------------- 2 files changed, 32 insertions(+), 65 deletions(-) diff --git a/emberregex/engine.mojo b/emberregex/engine.mojo index 4ecf18e..4d38f98 100644 --- a/emberregex/engine.mojo +++ b/emberregex/engine.mojo @@ -51,7 +51,6 @@ from .optimize import ( LiteralAlt, ) from .teddy import ( - teddy_find_prefix, teddy_full_match, teddy_match_at, teddy_search_forward, @@ -3646,7 +3645,9 @@ struct Regex[pattern: String](Copyable, Movable): comptime if Self._strategy.fprefix_len > 0: return self._find_prefix_candidate(input, input_len, pos) else: - return teddy_find_prefix[alt=Self._alt_prefix](input, pos) + return teddy_search_forward[alt=Self._alt_prefix, want_end=False]( + input, pos + )[0] @always_inline def _find_prefix_candidate[ diff --git a/emberregex/teddy.mojo b/emberregex/teddy.mojo index 37be2cd..3a47a0e 100644 --- a/emberregex/teddy.mojo +++ b/emberregex/teddy.mojo @@ -26,11 +26,9 @@ requires HAS_FAST_BYTE_SHUFFLE (see simd_kernels.mojo). from std.sys import simd_width_of from .optimize import LiteralAlt -from .simd_kernels import NIBBLE_TABLE_SIZE, nibble_lookup +from .simd_kernels import _NibbleTable, nibble_lookup from .simd_scan import clear_first_lane, first_lane_index, lane_bits -comptime _NibbleTable = SIMD[DType.uint8, NIBBLE_TABLE_SIZE] - def _teddy_pos_masks( alt: LiteralAlt, j: Int @@ -104,64 +102,18 @@ def teddy_full_match[ return False -@always_inline -def teddy_find_prefix[ - origin: Origin, //, alt: LiteralAlt -](input: Span[Byte, origin], start: Int) -> Int: - """First position >= start where any of the alternation's literal - chains occurs, or -1. Prefilter twin of teddy_search_forward for - patterns whose *required prefix* is a literal alternation - (`(?:GET|POST|PUT) /...`): the caller runs the real engine at each - returned candidate.""" - comptime W = simd_width_of[DType.uint8]() - comptime k = min(3, alt.min_len) - comptime m0 = _teddy_pos_masks(alt, 0) - comptime m1 = _teddy_pos_masks(alt, 1 if k > 1 else 0) - comptime m2 = _teddy_pos_masks(alt, 2 if k > 2 else 0) - - var input_len = len(input) - var pos = start - var ptr = Pointer(input.unsafe_ptr()) - - while pos + W <= input_len: - var v = ptr.unsafe_offset(pos).unsafe_load[width=W]() - var lo = v & 0x0F - var hi = v >> 4 - var cand = nibble_lookup(m0[0], lo) & nibble_lookup(m0[1], hi) - comptime if k > 1: - var c1 = nibble_lookup(m1[0], lo) & nibble_lookup(m1[1], hi) - cand &= c1.shift_left[1]() - comptime if k > 2: - var c2 = nibble_lookup(m2[0], lo) & nibble_lookup(m2[1], hi) - cand &= c2.shift_left[2]() - var bits = lane_bits(cand.ne(0)) - while bits != 0: - var at = pos + first_lane_index(bits) - comptime for i in range(len(alt.lits)): - comptime lit = alt.lits[i].copy() - comptime cli = alt.caseless[i].copy() - if _lit_at[lit=lit, cl=cli](input, at): - return at - bits = clear_first_lane(bits) - pos += W - (k - 1) - - while pos + alt.min_len <= input_len: - comptime for i in range(len(alt.lits)): - comptime lit = alt.lits[i].copy() - comptime cli = alt.caseless[i].copy() - if _lit_at[lit=lit, cl=cli](input, pos): - return pos - pos += 1 - - return -1 - - @always_inline def teddy_search_forward[ - origin: Origin, //, alt: LiteralAlt + origin: Origin, //, alt: LiteralAlt, want_end: Bool = True ](input: Span[Byte, origin], start: Int) -> Tuple[Int, Int]: """First match from `start` as (start, leftmost-longest end), or - (-1, -1) (mirrors the DFA-lane search_forward contract).""" + (-1, -1) (mirrors the DFA-lane search_forward contract). + + `want_end=False` is the prefilter form, for patterns whose *required + prefix* is a literal alternation (`(?:GET|POST|PUT) /...`): the first + position where any literal occurs, verified by the first literal that + fits rather than all of them, with the end left at -1 — the caller + runs the real engine at each returned candidate.""" comptime W = simd_width_of[DType.uint8]() comptime k = min(3, alt.min_len) comptime m0 = _teddy_pos_masks(alt, 0) @@ -186,18 +138,32 @@ def teddy_search_forward[ var bits = lane_bits(cand.ne(0)) while bits != 0: var at = pos + first_lane_index(bits) - var end = teddy_match_at[alt=alt](input, at) - if end >= 0: - return (at, end) + comptime if want_end: + var end = teddy_match_at[alt=alt](input, at) + if end >= 0: + return (at, end) + else: + comptime for i in range(len(alt.lits)): + comptime lit = alt.lits[i].copy() + comptime cli = alt.caseless[i].copy() + if _lit_at[lit=lit, cl=cli](input, at): + return (at, -1) bits = clear_first_lane(bits) # The last k-1 lanes were masked off by the zero-filling lane # shifts; rescan them as the head of the next chunk. pos += W - (k - 1) while pos + alt.min_len <= input_len: - var end = teddy_match_at[alt=alt](input, pos) - if end >= 0: - return (pos, end) + comptime if want_end: + var end = teddy_match_at[alt=alt](input, pos) + if end >= 0: + return (pos, end) + else: + comptime for i in range(len(alt.lits)): + comptime lit = alt.lits[i].copy() + comptime cli = alt.caseless[i].copy() + if _lit_at[lit=lit, cl=cli](input, pos): + return (pos, -1) pos += 1 return (-1, -1) From 4d7fe53c14ba94858aeb0dcf0f8e382365f3b69f Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:13:28 -0600 Subject: [PATCH 41/65] cleanup: one tail loop in simd_find_literal_rare The exact- and caseless-first-byte tails differed only in hopping between first-byte hits with simd_find_byte (and skipping byte 0 in the verify). Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/simd_scan.mojo | 46 +++++++++++++++------------------------ 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/emberregex/simd_scan.mojo b/emberregex/simd_scan.mojo index e54163c..490218b 100644 --- a/emberregex/simd_scan.mojo +++ b/emberregex/simd_scan.mojo @@ -292,40 +292,28 @@ def simd_find_literal_rare[ # where the tail dominates); a caseless first byte falls back to the # per-position verify. comptime c0 = cl[0] - comptime if not c0: - comptime fb = lit[0] - while True: + # An exact first byte is already verified by the hop. + comptime j0 = 0 if c0 else 1 + while True: + comptime if not c0: + comptime fb = lit[0] var candidate = simd_find_byte(input, fb, pos) if candidate < 0: return -1 pos = candidate - if pos + n > input_len: - return -1 - var ok = True - comptime for j in range(1, n): - comptime cj = cl[j] - comptime bj = lit[j] - if ok: - ok = probe_eq1[caseless=cj, target=bj]( - input.unsafe_get(pos + j) - ) - if ok: - return pos - pos += 1 - else: - while pos + n <= input_len: - var ok = True - comptime for j in range(n): - comptime cj = cl[j] - comptime bj = lit[j] - if ok: - ok = probe_eq1[caseless=cj, target=bj]( - input.unsafe_get(pos + j) - ) + if pos + n > input_len: + return -1 + var ok = True + comptime for j in range(j0, n): + comptime cj = cl[j] + comptime bj = lit[j] if ok: - return pos - pos += 1 - return -1 + ok = probe_eq1[caseless=cj, target=bj]( + input.unsafe_get(pos + j) + ) + if ok: + return pos + pos += 1 def simd_find_literal[ From af95e79ff7c864334d98ed9acc057acaf0075606 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:16:06 -0600 Subject: [PATCH 42/65] cleanup: build_lf_dfa returns EagerDFA; drop LFDFA and prev_ids LFDFA.valid always equalled d.valid (_edfa_finish sets it) and prev_ids only fed one structural test; the look-behind states it appended to `starts` were only renumbered, never read by the finish, so the table is unchanged. That test is deleted: the invariant lives in the builder's look-behind lane, not in the table, and test_both_class_atom_before_anchor / test_differential_both_class_atoms pin the same fix (bd077fe) through the directly built LF table against the Pike VM. build_lf_dfa's minimize hook had no other user. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/engine.mojo | 24 ++++++------ emberregex/static_lfdfa.mojo | 50 +++++-------------------- test/test_leftmost_first_dfa.mojo | 28 +++++++------- test/test_word_boundary_dfa.mojo | 62 +++++++------------------------ 4 files changed, 48 insertions(+), 116 deletions(-) diff --git a/emberregex/engine.mojo b/emberregex/engine.mojo index 4d38f98..6747056 100644 --- a/emberregex/engine.mojo +++ b/emberregex/engine.mojo @@ -1295,7 +1295,7 @@ struct Regex[pattern: String](Copyable, Movable): and not ( Self.nfa.has_word_boundary and Self._use_scan_filter - and not _edfa_has_region(Self._lfdfa.d) + and not _edfa_has_region(Self._lfdfa) ) ) # The DFA-bounded capture lane (Rust regex's meta "Core" strategy): @@ -1312,7 +1312,7 @@ struct Regex[pattern: String](Copyable, Movable): and not ( Self.nfa.has_word_boundary and Self._use_scan_filter - and not _edfa_has_region(Self._lfdfa.d) + and not _edfa_has_region(Self._lfdfa) ) ) comptime _use_lf_lane = Self._use_lf_dfa or Self._use_dfa_span @@ -1329,7 +1329,7 @@ struct Regex[pattern: String](Copyable, Movable): comptime _span_lane = Self._group_count > 0 comptime _use_lf_sheng = ( Self._use_lf_lane - and sheng_viable(Self._lfdfa.d) + and sheng_viable(Self._lfdfa) and HAS_FAST_BYTE_SHUFFLE ) # LazyDFA only backs DFA patterns whose CLASSIC comptime @@ -1363,17 +1363,17 @@ struct Regex[pattern: String](Copyable, Movable): ) # Leftmost-first lane tables (same materialization rules as above). comptime _SHENG_MASKS = static_bytes[Self._SHENG_MASKS_S]() - comptime _LFDFA_TN = edfa_table_len(Self._lfdfa.d.num_states) - comptime _LFDFA_DT = edfa_id_dtype(Self._lfdfa.d.num_states) + comptime _LFDFA_TN = edfa_table_len(Self._lfdfa.num_states) + comptime _LFDFA_DT = edfa_id_dtype(Self._lfdfa.num_states) comptime _LFDFA_TABLE_S = edfa_table_str[Self._LFDFA_TN, Self._LFDFA_DT]( - Self._lfdfa.d + Self._lfdfa ) - comptime _LFDFA_FLAGS = edfa_flags_arr[Self._lfdfa.d.num_states]( - Self._lfdfa.d + comptime _LFDFA_FLAGS = edfa_flags_arr[Self._lfdfa.num_states]( + Self._lfdfa ) - comptime _LF_SHENG_CAP = sheng_cap_for(Self._lfdfa.d, Self._use_lf_sheng) + comptime _LF_SHENG_CAP = sheng_cap_for(Self._lfdfa, Self._use_lf_sheng) comptime _LF_SHENG_MASKS_S = sheng_masks_str[Self._LF_SHENG_CAP]( - Self._lfdfa.d, Self._use_lf_sheng + Self._lfdfa, Self._use_lf_sheng ) comptime _LFDFA_TABLE = static_bytes[Self._LFDFA_TABLE_S]() comptime _LF_SHENG_MASKS = static_bytes[Self._LF_SHENG_MASKS_S]() @@ -1624,14 +1624,14 @@ struct Regex[pattern: String](Copyable, Movable): """Leftmost-first match END at or after `pos`, or -1.""" comptime if Self._use_lf_sheng: return sheng_match_at[ - d=Self._lfdfa.d, + d=Self._lfdfa, cap=Self._LF_SHENG_CAP, masks=Self._LF_SHENG_MASKS, flags=Self._LFDFA_FLAGS, ](input, pos) else: return edfa_match_at[ - d=Self._lfdfa.d, + d=Self._lfdfa, table=Self._LFDFA_TABLE, flags=Self._LFDFA_FLAGS, ](input, pos) diff --git a/emberregex/static_lfdfa.mojo b/emberregex/static_lfdfa.mojo index a5f4ee3..8d7816b 100644 --- a/emberregex/static_lfdfa.mojo +++ b/emberregex/static_lfdfa.mojo @@ -104,28 +104,6 @@ comptime _LFClo = SIMD[DType.int16, _LF_CLO_W] comptime _LF_SIG_BITS = 56 -struct LFDFA(Copyable, Movable): - """Comptime-computed leftmost-first DFA. - - `d` is the table in EagerDFA form — the same walkers, acceleration - data, flag bytes and Sheng masks apply — with its `start_*` fields - holding the UNANCHORED start states (restart bit set). - """ - - var valid: Bool - var d: EagerDFA - # Debug/test view: the ids (in `d`'s numbering) of the states whose - # look-behind class is "word". Exact only when built with - # `minimize=False` (minimization may merge such a state with an - # equivalent one entered on other bytes). - var prev_ids: List[Int] - - def __init__(out self): - self.valid = False - self.d = EagerDFA() - self.prev_ids = List[Int]() - - def _lf_closure( kinds: List[Int], out1s: List[Int], @@ -251,13 +229,14 @@ def _lf_memo_closure( return len(clo_vec) - 1 -def build_lf_dfa( - nfa: NFA, - enabled: Bool, - minimize: Bool = True, -) -> LFDFA: +def build_lf_dfa(nfa: NFA, enabled: Bool) -> EagerDFA: """Leftmost-first subset construction — runs at compile time. + The table comes back in EagerDFA form — the same walkers, + acceleration data, flag bytes and Sheng masks apply — with its + `start_*` fields holding the UNANCHORED start states (restart bit + set). + Returns an invalid placeholder when `enabled` is False, when the NFA cannot be bitset-indexed, when some state's ordered list would exceed the lane capacity, or when the state count exceeds EDFA_STATE_CAP @@ -281,10 +260,8 @@ def build_lf_dfa( closure, and while a state has at most _LF_SIG_BITS consuming members, "does member k accept class c" is a bit of the class's signature word rather than a List read. - - `minimize` is a test hook, as in `build_eager_dfa`. """ - var result = LFDFA() + var result = EagerDFA() if not enabled: return result^ var n = len(nfa.states) @@ -1072,18 +1049,9 @@ def build_lf_dfa( if st_selfloop[s] and not st_genuine[s]: flags[s] |= Int(EDFA_NO_ACCEL) - # The look-behind-"word" states ride along in `starts` so the finish - # (minimization remap + match permutation) renumbers them too. - for s in range(len(st_list)): - if Int(st_list[s][_LF_PREV_LANE]) != 0: - starts.append(s) - var pstarts = _edfa_finish( - result.d, rows, flags, starts, rep_lo, rep_hi, nclasses, minimize, nctx + result, rows, flags, starts, rep_lo, rep_hi, nclasses, True, nctx ) - for k in range(nctx, len(pstarts)): - result.prev_ids.append(pstarts[k]) if has_wb: - result.d.start_other_word = pstarts[3] - result.valid = True + result.start_other_word = pstarts[3] return result^ diff --git a/test/test_leftmost_first_dfa.mojo b/test/test_leftmost_first_dfa.mojo index af2ee2f..010784d 100644 --- a/test/test_leftmost_first_dfa.mojo +++ b/test/test_leftmost_first_dfa.mojo @@ -115,7 +115,7 @@ def test_tail_kind_shared_without_bol_multiline() raises: # restarting state (114 states, past the cap for the {6} sibling). comptime S = Regex["(?:a|b|\n)*a(?:a|b|\n){5}"] assert_true(S._lfdfa.valid) - assert_equal(S._lfdfa.d.num_states, 96) + assert_equal(S._lfdfa.num_states, 96) assert_true(S._use_lf_dfa) var re = S() var input = String("b\nab") + "a" * 3 + "\n" + "b" * 5 + "a" @@ -136,8 +136,8 @@ def test_spurious_self_loops_are_not_accelerated() raises: "|lap|lab|mop|mob|net|nap|owl|oak|pin|pit|rat|rib|sun|sit|tap|[0-9]{3}" ] assert_true(A._use_lf_dfa) - comptime a_accel = len(A._lfdfa.d.accel.states) + len( - A._lfdfa.d.accel.nib_states + comptime a_accel = len(A._lfdfa.accel.states) + len( + A._lfdfa.accel.nib_states ) assert_equal(a_accel, 1) # A genuine single-byte loop (the `a+` run) IS accelerated, on both @@ -145,8 +145,8 @@ def test_spurious_self_loops_are_not_accelerated() raises: # 20 KB run measured 16x slower when a loop-set threshold dropped it. comptime B = Regex["a+e|x"] assert_true(B._use_lf_dfa) - comptime b_lf_accel = len(B._lfdfa.d.accel.states) + len( - B._lfdfa.d.accel.nib_states + comptime b_lf_accel = len(B._lfdfa.accel.states) + len( + B._lfdfa.accel.nib_states ) comptime b_classic_accel = len(B._edfa.accel.states) + len( B._edfa.accel.nib_states @@ -232,7 +232,7 @@ def test_lazy_stops_at_first_close() raises: # Structural proof that the scan stops: the state after the first # `>` is a match state with no live transition, so the walker # returns there instead of walking the rest of the line. - comptime dead = _lf_row_dead_after(S._lfdfa.d, "") + comptime dead = _lf_row_dead_after(S._lfdfa, "") assert_true(dead) var re2 = Regex["x*?y"]() assert_equal(re2.search("xxxyxy").end, 4) @@ -276,7 +276,7 @@ def test_tiny_tables_materialize_as_shared_data() raises: # materialized arrays are padded to that size, so the constant is a # shared global and a short walk costs per-byte work only. comptime S = Regex["<.*?>"] - comptime assert S._lfdfa.d.num_states * 256 < EDFA_TABLE_MIN_BYTES + comptime assert S._lfdfa.num_states * 256 < EDFA_TABLE_MIN_BYTES comptime assert S._LFDFA_TN >= EDFA_TABLE_MIN_BYTES comptime assert S._RDFA_TN >= EDFA_TABLE_MIN_BYTES comptime assert S._EDFA_TN >= EDFA_TABLE_MIN_BYTES @@ -685,17 +685,17 @@ def test_wide_list_signature_renumbering() raises: # lists wider than the signature bitstring. comptime lf = build_lf_dfa(Regex[_WIDE_ALT].nfa, True) assert_true(lf.valid) - comptime tn = lf.d.num_states * 256 - comptime dt = edfa_id_dtype(lf.d.num_states) - comptime table = static_bytes[edfa_table_str[tn, dt](lf.d)]() - comptime flags = edfa_flags_arr[lf.d.num_states](lf.d) + comptime tn = lf.num_states * 256 + comptime dt = edfa_id_dtype(lf.num_states) + comptime table = static_bytes[edfa_table_str[tn, dt](lf)]() + comptime flags = edfa_flags_arr[lf.num_states](lf) var input = String("!!!!!!!!!!!!!!!!!!!!q!!") var bytes = input.as_bytes() - assert_equal(edfa_match_at[d=lf.d, table=table, flags=flags](bytes, 0), 21) - assert_equal(edfa_match_at[d=lf.d, table=table, flags=flags](bytes, 21), -1) + assert_equal(edfa_match_at[d=lf, table=table, flags=flags](bytes, 0), 21) + assert_equal(edfa_match_at[d=lf, table=table, flags=flags](bytes, 21), -1) var input2 = String("!!!!!!!!!!!!!!!!!!!!7!!") var bytes2 = input2.as_bytes() - assert_equal(edfa_match_at[d=lf.d, table=table, flags=flags](bytes2, 0), 21) + assert_equal(edfa_match_at[d=lf, table=table, flags=flags](bytes2, 0), 21) # And on the engine: a class arm keeps Teddy off, so the same lists # drive search/findall through the lane. comptime W = Regex[_WIDE_ALT + "|[!?]{2}"] diff --git a/test/test_word_boundary_dfa.mojo b/test/test_word_boundary_dfa.mojo index 061ed42..712371b 100644 --- a/test/test_word_boundary_dfa.mojo +++ b/test/test_word_boundary_dfa.mojo @@ -19,7 +19,6 @@ from emberregex.static_dfa import ( EDFA_MATCH_IF_WORD, EagerDFA, _edfa_has_region, - _is_word_byte, build_eager_dfa, edfa_flags_arr, edfa_full_match, @@ -27,7 +26,7 @@ from emberregex.static_dfa import ( edfa_match_at, edfa_table_str, ) -from emberregex.static_lfdfa import LFDFA, build_lf_dfa +from emberregex.static_lfdfa import build_lf_dfa from emberregex.static_rdfa import ( build_reverse_dfa, rdfa_find_start, @@ -70,7 +69,7 @@ def test_scanner_patterns_without_a_region_keep_the_backtracker() raises: assert_true(S._strategy.use_dfa) assert_true(S._strategy.use_eager_dfa) assert_true(S._use_scan_filter) - assert_false(comptime (_edfa_has_region(S._lfdfa.d))) + assert_false(comptime (_edfa_has_region(S._lfdfa))) assert_false(S._use_lf_dfa) assert_false(S._use_lazy_dfa) var re = S() @@ -89,10 +88,10 @@ def test_scanner_patterns_without_a_region_keep_the_backtracker() raises: # 'f'. comptime T = Regex["\\bfoo(?:bar|baz)\\b"] assert_true(T._use_scan_filter) - assert_true(comptime (_edfa_has_region(T._lfdfa.d))) + assert_true(comptime (_edfa_has_region(T._lfdfa))) assert_true(T._use_lf_dfa) comptime U = Regex["\\b(?:foo|bar)\\b"] - assert_true(comptime (_edfa_has_region(U._lfdfa.d))) + assert_true(comptime (_edfa_has_region(U._lfdfa))) assert_true(U._use_lf_dfa) @@ -128,19 +127,19 @@ def test_classic_tables_unchanged_for_anchor_free_patterns() raises: # word anchor exists. comptime A = Regex["[a-z]+://[a-z.]+"] assert_equal(A._edfa.num_states, 6) - assert_equal(A._lfdfa.d.num_states, 6) + assert_equal(A._lfdfa.num_states, 6) assert_equal(A._rdfa.num_states, 6) comptime B = Regex["(?:foo|bar|ba+z)+"] assert_equal(B._edfa.num_states, 7) - assert_equal(B._lfdfa.d.num_states, 12) + assert_equal(B._lfdfa.num_states, 12) assert_equal(B._rdfa.num_states, 8) comptime C = Regex["(?m)^(?:ab|cd)$"] assert_equal(C._edfa.num_states, 5) - assert_equal(C._lfdfa.d.num_states, 5) + assert_equal(C._lfdfa.num_states, 5) assert_equal(C._rdfa.num_states, 5) # ...and their mid-line start states do not split by word class. assert_equal(A._edfa.start_other_word, A._edfa.start_other) - assert_equal(B._lfdfa.d.start_other_word, B._lfdfa.d.start_other) + assert_equal(B._lfdfa.start_other_word, B._lfdfa.start_other) assert_equal(C._rdfa.seed_other_word, C._rdfa.seed_other) assert_false(A._edfa.any_wb) @@ -191,41 +190,6 @@ def test_both_classes_fold_into_a_plain_match() raises: assert_false(d.any_wb) -def _prev_states_entered_on_word_bytes(lf: LFDFA) -> Bool: - """Comptime: the table has look-behind-"word" states and every - transition INTO one is on a word byte (built unminimized, so the ids - are exact).""" - if not lf.valid or len(lf.prev_ids) == 0: - return False - var n = lf.d.num_states - for t in lf.prev_ids: - for s in range(n): - for b in range(256): - if lf.d.table[s * 256 + b] == t and not _is_word_byte(b): - return False - return True - - -def _lf_prev_invariant[p: StaticString]() -> Bool: - comptime nfa = Regex[p].nfa - comptime lf = build_lf_dfa(nfa, True, minimize=False) - comptime ok = _prev_states_entered_on_word_bytes(lf) - return ok - - -def test_lf_prev_states_entered_on_word_bytes_only() raises: - # Structural form of the look-behind invariant, on the patterns whose - # pending anchor follows a both-class atom and on the simple shapes. - assert_true(_lf_prev_invariant["\\bfoo\\b"]()) - assert_true(_lf_prev_invariant[".\\b.|q"]()) - assert_true(_lf_prev_invariant["\\S+\\bing\\b|q"]()) - assert_true(_lf_prev_invariant["[\\w.-]+\\bfoo|q"]()) - assert_true(_lf_prev_invariant["(?s)[a-z .]\\b\\w+|q"]()) - assert_true(_lf_prev_invariant["\\b(?:foo|bar)\\b"]()) - assert_true(_lf_prev_invariant["\\w+\\b|q"]()) - assert_true(_lf_prev_invariant["(?:ab\\B)+c"]()) - - # --- Direct table harness --------------------------------------------------- @@ -243,10 +207,10 @@ def _forced_lane_check[p: StaticString](input: String, label: String) raises: comptime efl = edfa_flags_arr[ed.num_states](ed) comptime lf = build_lf_dfa(nfa, True) comptime assert lf.valid - comptime LTN = lf.d.num_states * 256 - comptime LDT = edfa_id_dtype(lf.d.num_states) - comptime ltbl = static_bytes[edfa_table_str[LTN, LDT](lf.d)]() - comptime lfl = edfa_flags_arr[lf.d.num_states](lf.d) + comptime LTN = lf.num_states * 256 + comptime LDT = edfa_id_dtype(lf.num_states) + comptime ltbl = static_bytes[edfa_table_str[LTN, LDT](lf)]() + comptime lfl = edfa_flags_arr[lf.num_states](lf) comptime rd = build_reverse_dfa(nfa, True) comptime assert rd.valid comptime RTN = rd.num_states * 256 @@ -274,7 +238,7 @@ def _forced_lane_check[p: StaticString](input: String, label: String) raises: var pos = 0 var i = 0 while pos <= n: - var end = edfa_match_at[d=lf.d, table=ltbl, flags=lfl]( + var end = edfa_match_at[d=lf, table=ltbl, flags=lfl]( bytes, pos ) if end < 0: From 5d1b4b0bd814a1427f9bd35c28228e89aa05e031 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:16:32 -0600 Subject: [PATCH 43/65] cleanup: stale table-name docstrings (edfa_table_arr, rdfa, onepass_table_arr) Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/onepass.mojo | 3 ++- emberregex/static_dfa.mojo | 2 +- emberregex/static_rdfa.mojo | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/emberregex/onepass.mojo b/emberregex/onepass.mojo index 1c7e236..fb2a5e9 100644 --- a/emberregex/onepass.mojo +++ b/emberregex/onepass.mojo @@ -67,7 +67,8 @@ passes keeps the caller's -1. Tables (POD + Array rule: the struct crosses into the walkers as a comptime parameter, the bulk as separate arrays padded to at least EDFA_TABLE_MIN_BYTES so they lower to shared constant data): -`onepass_table_arr` — `num_states x nclasses` Int32 cells, -1 dead, else +`onepass_table_str` — `num_states x nclasses` Int32 cells (a string +literal, see static_bytes.mojo), -1 dead, else the premultiplied next row, the next state id and the slot-set id packed (`_OP_*` shifts); `onepass_class_arr` — byte to class; `onepass_eps_arr` — slot bitsets by id (id 0 is the empty set); `onepass_state_arr` — per diff --git a/emberregex/static_dfa.mojo b/emberregex/static_dfa.mojo index dc3c7a0..85f8104 100644 --- a/emberregex/static_dfa.mojo +++ b/emberregex/static_dfa.mojo @@ -582,7 +582,7 @@ struct EagerDFA(Copyable, Movable): """Comptime-computed DFA: flat transition table + per-state flags. Only ever exists as a comptime value; the runtime engine reads the - materialized Array forms (see edfa_table_arr / edfa_flags_arr). + materialized forms (see edfa_table_str / edfa_flags_arr). """ var valid: Bool diff --git a/emberregex/static_rdfa.mojo b/emberregex/static_rdfa.mojo index 1ff3346..f96589b 100644 --- a/emberregex/static_rdfa.mojo +++ b/emberregex/static_rdfa.mojo @@ -99,7 +99,8 @@ comptime RDFA_STATE_CAP = _MIN_CAP struct RDFA(Copyable, Movable): """Comptime-computed reverse DFA. Only ever exists as a comptime - value; the walker reads the materialized Array forms.""" + value; the walker reads the materialized forms (rdfa_table_str / + rdfa_flags_arr).""" var valid: Bool var num_states: Int From 548eff9ce19e70bd5ef471eed91cfea8b02ffd83 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:17:17 -0600 Subject: [PATCH 44/65] cleanup: format Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/engine.mojo | 4 +--- emberregex/static_dfa.mojo | 6 +++--- test/test_word_boundary_dfa.mojo | 4 +--- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/emberregex/engine.mojo b/emberregex/engine.mojo index 6747056..9eaee13 100644 --- a/emberregex/engine.mojo +++ b/emberregex/engine.mojo @@ -1368,9 +1368,7 @@ struct Regex[pattern: String](Copyable, Movable): comptime _LFDFA_TABLE_S = edfa_table_str[Self._LFDFA_TN, Self._LFDFA_DT]( Self._lfdfa ) - comptime _LFDFA_FLAGS = edfa_flags_arr[Self._lfdfa.num_states]( - Self._lfdfa - ) + comptime _LFDFA_FLAGS = edfa_flags_arr[Self._lfdfa.num_states](Self._lfdfa) comptime _LF_SHENG_CAP = sheng_cap_for(Self._lfdfa, Self._use_lf_sheng) comptime _LF_SHENG_MASKS_S = sheng_masks_str[Self._LF_SHENG_CAP]( Self._lfdfa, Self._use_lf_sheng diff --git a/emberregex/static_dfa.mojo b/emberregex/static_dfa.mojo index 85f8104..21dc601 100644 --- a/emberregex/static_dfa.mojo +++ b/emberregex/static_dfa.mojo @@ -2467,9 +2467,9 @@ def edfa_match_at[ return _edfa_walk_impl[ d=d, table=table, flags=flags, accel=True, cap=cap ](input, start) - return _edfa_walk_impl[ - d=d, table=table, flags=flags, accel=False, cap=cap - ](input, start) + return _edfa_walk_impl[d=d, table=table, flags=flags, accel=False, cap=cap]( + input, start + ) @always_inline diff --git a/test/test_word_boundary_dfa.mojo b/test/test_word_boundary_dfa.mojo index 712371b..6f95eeb 100644 --- a/test/test_word_boundary_dfa.mojo +++ b/test/test_word_boundary_dfa.mojo @@ -238,9 +238,7 @@ def _forced_lane_check[p: StaticString](input: String, label: String) raises: var pos = 0 var i = 0 while pos <= n: - var end = edfa_match_at[d=lf, table=ltbl, flags=lfl]( - bytes, pos - ) + var end = edfa_match_at[d=lf, table=ltbl, flags=lfl](bytes, pos) if end < 0: break var start = rdfa_find_start[d=rd, table=rtbl, flags=rfl]( From 80c9fc5a9c807de969be28dcf887b6fafce49443 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:32:47 -0600 Subject: [PATCH 45/65] cleanup: pick the LF hash-lookup lane slices with a comptime for Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/static_lfdfa.mojo | 64 ++++++++++++++---------------------- 1 file changed, 24 insertions(+), 40 deletions(-) diff --git a/emberregex/static_lfdfa.mojo b/emberregex/static_lfdfa.mojo index 73b7fdd..3b9b890 100644 --- a/emberregex/static_lfdfa.mojo +++ b/emberregex/static_lfdfa.mojo @@ -516,26 +516,17 @@ def build_lf_dfa( acc_h ^= _WB_PREV_SALT var found = -1 var eqm = hashv.eq(SIMD[DType.uint64, 256](acc_h)) - for j in range(4): - if found >= 0: - break - var word: UInt64 - if j == 0: - word = _lane_word(eqm.slice[64, offset=0]()) - elif j == 1: - word = _lane_word(eqm.slice[64, offset=64]()) - elif j == 2: - word = _lane_word(eqm.slice[64, offset=128]()) - else: - word = _lane_word(eqm.slice[64, offset=192]()) - while word != 0: - var cand = 64 * j + Int(count_trailing_zeros(word)) - word &= word - 1 - if cand >= len(st_list): - break - if ((st_list.unsafe_get(cand) ^ acc).reduce_or()) == 0: - found = cand - break + comptime for j in range(4): + if found < 0: + var word = _lane_word(eqm.slice[64, offset=64 * j]()) + while word != 0: + var cand = 64 * j + Int(count_trailing_zeros(word)) + word &= word - 1 + if cand >= len(st_list): + break + if ((st_list.unsafe_get(cand) ^ acc).reduce_or()) == 0: + found = cand + break if found < 0: found = len(st_list) hashv[found] = acc_h @@ -1019,26 +1010,19 @@ def build_lf_dfa( acc_h ^= _WB_PREV_SALT var found = -1 var eqm = hashv.eq(SIMD[DType.uint64, 256](acc_h)) - for j in range(4): - if found >= 0: - break - var word: UInt64 - if j == 0: - word = _lane_word(eqm.slice[64, offset=0]()) - elif j == 1: - word = _lane_word(eqm.slice[64, offset=64]()) - elif j == 2: - word = _lane_word(eqm.slice[64, offset=128]()) - else: - word = _lane_word(eqm.slice[64, offset=192]()) - while word != 0: - var cand = 64 * j + Int(count_trailing_zeros(word)) - word &= word - 1 - if cand >= len(st_list): - break - if ((st_list.unsafe_get(cand) ^ acc).reduce_or()) == 0: - found = cand - break + comptime for j in range(4): + if found < 0: + var word = _lane_word(eqm.slice[64, offset=64 * j]()) + while word != 0: + var cand = 64 * j + Int(count_trailing_zeros(word)) + word &= word - 1 + if cand >= len(st_list): + break + if ( + (st_list.unsafe_get(cand) ^ acc).reduce_or() + ) == 0: + found = cand + break if found < 0: if len(st_list) >= EDFA_STATE_CAP + 1: return result^ # state blowup: stay invalid From 5f5d38ecde026e7b625b1b36ae38fa71cb0442bc Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:46:02 -0600 Subject: [PATCH 46/65] cleanup: share the determinizer setup through _FlatNFA and small helpers _flatten_nfa's 13 arguments become a _FlatNFA built in place (its constructor also runs _byte_classes); _class_ranges, _word_classes, _eol_ok_bits and _bs_members replace the per-builder copies of the rep_lo/rep_hi, word-class, pending-EOL and bitset->member-list loops. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/onepass.mojo | 44 ++---- emberregex/set_dfa.mojo | 61 ++----- emberregex/set_reverse.mojo | 56 ++----- emberregex/static_dfa.mojo | 298 ++++++++++++++++++++--------------- emberregex/static_lfdfa.mojo | 80 +++------- emberregex/static_rdfa.mojo | 56 ++----- test/test_eager_dfa.mojo | 91 ++--------- 7 files changed, 263 insertions(+), 423 deletions(-) diff --git a/emberregex/onepass.mojo b/emberregex/onepass.mojo index c9a4f2a..0e41502 100644 --- a/emberregex/onepass.mojo +++ b/emberregex/onepass.mojo @@ -93,11 +93,10 @@ from .static_dfa import ( EDFA_TABLE_MIN_BYTES, EagerDFA, _StateBits, + _FlatNFA, _bs_set, - _byte_classes, _edfa_accel_skip, _edfa_has_accel, - _flatten_nfa, _is_word_byte, _nfa_has_word_anchor, _wb_holds, @@ -341,34 +340,16 @@ def build_onepass(nfa: NFA, enabled: Bool) -> OnePass: var has_wb = _nfa_has_word_anchor(nfa) # Byte classes and the flat NFA views (shared with the eager DFAs). - var class_of = List[Int](fill=0, length=256) - var reps = _byte_classes(nfa, class_of) - var nclasses = len(reps) - var nl_class = class_of[Int(CHAR_NEWLINE)] - var kinds = List[Int]() - var out1s = List[Int]() - var out2s = List[Int]() - var anchors = List[Int]() - var cls_mask = List[SIMD[DType.uint64, 4]]() - var consuming_bits = _StateBits(0) - var match_bits = _StateBits(0) - var eol_bits = _StateBits(0) - var flat_has_bol_ml = False - _flatten_nfa( - nfa, - class_of, - nclasses, - nl_class, - kinds, - out1s, - out2s, - anchors, - cls_mask, - consuming_bits, - match_bits, - eol_bits, - flat_has_bol_ml, - ) + var flat = _FlatNFA(nfa) + ref class_of = flat.class_of + ref reps = flat.reps + var nclasses = flat.nclasses + var nl_class = flat.nl_class + ref kinds = flat.kinds + ref out1s = flat.out1s + ref out2s = flat.out2s + ref anchors = flat.anchors + ref cls_mask = flat.cls_mask var save_slots = List[Int]() for i in range(n): save_slots.append(nfa.states[i].save_slot) @@ -648,7 +629,8 @@ def build_onepass(nfa: NFA, enabled: Bool) -> OnePass: result.valid = True result.num_states = num_states result.nclasses = nclasses - result.class_of = class_of^ + result.class_of = flat.class_of^ + flat.class_of = List[Int]() # a moved-out field must be reinitialized result.trans_next = trans_next^ result.trans_eps = trans_eps^ result.eps_sets = eps_sets^ diff --git a/emberregex/set_dfa.mojo b/emberregex/set_dfa.mojo index ac392ba..440fc8a 100644 --- a/emberregex/set_dfa.mojo +++ b/emberregex/set_dfa.mojo @@ -52,13 +52,15 @@ from .simd_scan import first_lane_index, lane_bits from .static_bytes import table_bytes from .static_dfa import ( EDFA_NFA_CAP, + _FlatNFA, _StateBits, _bs_eq, _bs_hash, + _bs_members, _byte_classes, + _class_ranges, _find_exit2, _flat_closure, - _flatten_nfa, WB_DROP, ) @@ -314,44 +316,20 @@ def build_multi_dfa(nfa: NFA, enabled: Bool) -> MultiDFA: var n = len(nfa.states) - # --- Byte classes: intervals with a representative first byte. --- - var class_of = List[Int](fill=-1, length=256) - var reps = _byte_classes(nfa, class_of) - var nclasses = len(reps) + # --- Byte classes + one flat pass over the NFA (see _FlatNFA). --- + var flat = _FlatNFA(nfa) + var nclasses = flat.nclasses + var nl_class = flat.nl_class + ref kinds = flat.kinds + ref out1s = flat.out1s + ref out2s = flat.out2s + ref anchors = flat.anchors + ref cls_mask = flat.cls_mask + ref consuming_bits = flat.consuming_bits + var has_bol_ml = flat.has_bol_ml var rep_lo = SIMD[DType.int32, 256](0) var rep_hi = SIMD[DType.int32, 256](0) - for ci in range(nclasses): - rep_lo[ci] = Int32(reps[ci]) - rep_hi[ci] = Int32(reps[ci + 1] - 1) if ci + 1 < nclasses else Int32( - 255 - ) - var nl_class = class_of[Int(CHAR_NEWLINE)] - - # --- One flat pass over the NFA (see _flatten_nfa). --- - var kinds = List[Int]() - var out1s = List[Int]() - var out2s = List[Int]() - var anchors = List[Int]() - var cls_mask = List[SIMD[DType.uint64, 4]]() - var consuming_bits = _StateBits(0) - var match_bits = _StateBits(0) - var eol_bits = _StateBits(0) - var has_bol_ml = False - _flatten_nfa( - nfa, - class_of, - nclasses, - nl_class, - kinds, - out1s, - out2s, - anchors, - cls_mask, - consuming_bits, - match_bits, - eol_bits, - has_bol_ml, - ) + _class_ranges(flat.reps, rep_lo, rep_hi) # --- The folded unanchored restart, precomputed per context. --- var start_o = _flat_closure( @@ -461,14 +439,7 @@ def build_multi_dfa(nfa: NFA, enabled: Bool) -> MultiDFA: var n_sets = len(sets_bits) var sets = List[List[Int]]() for si in range(n_sets): - var bits = sets_bits.unsafe_get(si) - var members = List[Int]() - for l in range(64): - var w = bits[l] - while w != 0: - members.append(64 * l + Int(count_trailing_zeros(w))) - w &= w - 1 - sets.append(members^) + sets.append(_bs_members(sets_bits.unsafe_get(si))) return _mdfa_finish(nfa, sets, rows^, s0, result^) diff --git a/emberregex/set_reverse.mojo b/emberregex/set_reverse.mojo index 67bf96a..9450bd9 100644 --- a/emberregex/set_reverse.mojo +++ b/emberregex/set_reverse.mojo @@ -59,11 +59,13 @@ from .set_pike import SetSpan from .static_bytes import table_bytes from .static_dfa import ( EDFA_NFA_CAP, + _FlatNFA, _StateBits, _bs_any, + _bs_members, _bs_set, _byte_classes, - _flatten_nfa, + _class_ranges, _wb_holds, WB_DROP, WB_PENDING, @@ -452,42 +454,17 @@ def build_reverse_dfa(nfa: NFA, enabled: Bool) -> ReverseDFA: var preds = _reverse_edges(nfa) # --- Byte classes + flat views (shared with the forward builders). --- - var class_of = List[Int](fill=-1, length=256) - var reps = _byte_classes(nfa, class_of) - var nclasses = len(reps) + var flat = _FlatNFA(nfa) + var nclasses = flat.nclasses + var nl_class = flat.nl_class + ref kinds = flat.kinds + ref anchors = flat.anchors + ref cls_mask = flat.cls_mask + ref eol_bits = flat.eol_bits + var has_bol_ml = flat.has_bol_ml var rep_lo = SIMD[DType.int32, 256](0) var rep_hi = SIMD[DType.int32, 256](0) - for ci in range(nclasses): - rep_lo[ci] = Int32(reps[ci]) - rep_hi[ci] = Int32(reps[ci + 1] - 1) if ci + 1 < nclasses else Int32( - 255 - ) - var nl_class = class_of[Int(CHAR_NEWLINE)] - - var kinds = List[Int]() - var out1s = List[Int]() - var out2s = List[Int]() - var anchors = List[Int]() - var cls_mask = List[SIMD[DType.uint64, 4]]() - var consuming_bits = _StateBits(0) - var match_bits = _StateBits(0) - var eol_bits = _StateBits(0) - var has_bol_ml = False - _flatten_nfa( - nfa, - class_of, - nclasses, - nl_class, - kinds, - out1s, - out2s, - anchors, - cls_mask, - consuming_bits, - match_bits, - eol_bits, - has_bol_ml, - ) + _class_ranges(flat.reps, rep_lo, rep_hi) # Predecessors, flattened, plus a predecessor bitset per state (the # raw reverse step is then a union of per-member bitsets filtered by @@ -671,14 +648,7 @@ def build_reverse_dfa(nfa: NFA, enabled: Bool) -> ReverseDFA: var n_sets = len(sets_bits) var sets = List[List[Int]]() for si in range(n_sets): - var bits = sets_bits.unsafe_get(si) - var members = List[Int]() - for l in range(64): - var w = bits[l] - while w != 0: - members.append(64 * l + Int(count_trailing_zeros(w))) - w &= w - 1 - sets.append(members^) + sets.append(_bs_members(sets_bits.unsafe_get(si))) # One 256-lane vector store per row (see static_dfa `_edfa_finish`). var table = List[Int](fill=-1, length=n_sets * 256) for si in range(n_sets): diff --git a/emberregex/static_dfa.mojo b/emberregex/static_dfa.mojo index ceced9c..02bd6c4 100644 --- a/emberregex/static_dfa.mojo +++ b/emberregex/static_dfa.mojo @@ -825,81 +825,163 @@ def _byte_classes(nfa: NFA, mut class_of: List[Int]) -> List[Int]: return reps^ -def _flatten_nfa( - nfa: NFA, - class_of: List[Int], - nclasses: Int, - nl_class: Int, - mut kinds: List[Int], - mut out1s: List[Int], - mut out2s: List[Int], - mut anchors: List[Int], - mut cls_mask: List[SIMD[DType.uint64, 4]], - mut consuming_bits: _StateBits, - mut match_bits: _StateBits, - mut eol_bits: _StateBits, - mut has_bol_ml: Bool, +struct _FlatNFA: + """Byte classes plus one flat pass over the NFA — the setup every bitset + determinizer shares. Built ONCE per build, in place (`var f = + _FlatNFA(nfa)`); callers bind the fields they need to locals with `ref` + and pass those, never the struct, into the hot loops, which then never + carry an aggregate across a call boundary.""" + + var class_of: List[Int] # 256 entries: byte -> class (_byte_classes) + var reps: List[Int] # first byte of each class; classes are intervals + var nclasses: Int + var nl_class: Int # the class of '\n' + # Flat views for the closure walks. + var kinds: List[Int] + var out1s: List[Int] + var out2s: List[Int] + var anchors: List[Int] + # Each state's accepted-class mask (classes are byte intervals wholly + # inside or outside every accept set, so a class bitmask per state + # captures acceptance exactly). + var cls_mask: List[SIMD[DType.uint64, 4]] + var consuming_bits: _StateBits + var match_bits: _StateBits + var eol_bits: _StateBits # EOL / EOL_MULTILINE anchors + var has_bol_ml: Bool + + def __init__(out self, nfa: NFA): + self.class_of = List[Int](fill=-1, length=256) + self.reps = _byte_classes(nfa, self.class_of) + var nclasses = len(self.reps) + var nl_class = self.class_of[Int(CHAR_NEWLINE)] + self.nclasses = nclasses + self.nl_class = nl_class + self.kinds = List[Int]() + self.out1s = List[Int]() + self.out2s = List[Int]() + self.anchors = List[Int]() + self.cls_mask = List[SIMD[DType.uint64, 4]]() + self.consuming_bits = _StateBits(0) + self.match_bits = _StateBits(0) + self.eol_bits = _StateBits(0) + self.has_bol_ml = False + var n = len(nfa.states) + for i in range(n): + var kind = nfa.states[i].kind + self.kinds.append(kind) + self.out1s.append(nfa.states[i].out1) + self.out2s.append(nfa.states[i].out2) + var at = nfa.states[i].anchor_type + self.anchors.append(at) + var cm = SIMD[DType.uint64, 4](0) + if kind == NFAStateKind.CHAR: + var c = Int(nfa.states[i].char_value) + if c < 256: + var ci = self.class_of[c] + cm[ci >> 6] = cm[ci >> 6] | (UInt64(1) << UInt64(ci & 63)) + _bs_set(self.consuming_bits, i) + elif kind == NFAStateKind.ANY: + for ci in range(nclasses): + cm[ci >> 6] = cm[ci >> 6] | (UInt64(1) << UInt64(ci & 63)) + cm[nl_class >> 6] = cm[nl_class >> 6] & ~( + UInt64(1) << UInt64(nl_class & 63) + ) + _bs_set(self.consuming_bits, i) + elif kind == NFAStateKind.CHARSET: + var cs = nfa.states[i].charset_index + for r in range(len(nfa.charsets[cs].ranges)): + var lo = Int(nfa.charsets[cs].ranges[r].lo) + var hi = Int(nfa.charsets[cs].ranges[r].hi) + if lo > hi or lo > 255: + continue + if hi > 255: + hi = 255 + # _byte_classes marked lo and hi+1, so the classes of + # lo and hi bound exactly the classes inside [lo, hi]. + for ci in range(self.class_of[lo], self.class_of[hi] + 1): + cm[ci >> 6] = cm[ci >> 6] | ( + UInt64(1) << UInt64(ci & 63) + ) + if nfa.charsets[cs].negated: + # Classes are pure w.r.t. this charset, so negation is + # exact at class granularity. + for w in range(4): + cm[w] = ~cm[w] + for ci in range(nclasses, 256): + cm[ci >> 6] = cm[ci >> 6] & ~( + UInt64(1) << UInt64(ci & 63) + ) + _bs_set(self.consuming_bits, i) + elif kind == NFAStateKind.MATCH: + _bs_set(self.match_bits, i) + elif kind == NFAStateKind.ANCHOR: + if at == AnchorKind.EOL or at == AnchorKind.EOL_MULTILINE: + _bs_set(self.eol_bits, i) + elif at == AnchorKind.BOL_MULTILINE: + self.has_bol_ml = True + self.cls_mask.append(cm) + + +def _class_ranges( + reps: List[Int], + mut rep_lo: SIMD[DType.int32, 256], + mut rep_hi: SIMD[DType.int32, 256], ): - """One flat pass over the NFA, shared by the bitset determinizers. - - Fills flat views for the closure walks, membership bitsets, and each - state's accepted-class mask (classes are byte intervals wholly inside - or outside every accept set, so a class bitmask per state captures - acceptance exactly). Called ONCE per build: the point of the flat - views is that the hot loops never pass the NFA aggregate across a - call boundary again. - """ - var n = len(nfa.states) - for i in range(n): - var kind = nfa.states[i].kind - kinds.append(kind) - out1s.append(nfa.states[i].out1) - out2s.append(nfa.states[i].out2) - var at = nfa.states[i].anchor_type - anchors.append(at) - var cm = SIMD[DType.uint64, 4](0) - if kind == NFAStateKind.CHAR: - var c = Int(nfa.states[i].char_value) - if c < 256: - var ci = class_of[c] - cm[ci >> 6] = cm[ci >> 6] | (UInt64(1) << UInt64(ci & 63)) - _bs_set(consuming_bits, i) - elif kind == NFAStateKind.ANY: - for ci in range(nclasses): - cm[ci >> 6] = cm[ci >> 6] | (UInt64(1) << UInt64(ci & 63)) - cm[nl_class >> 6] = cm[nl_class >> 6] & ~( - UInt64(1) << UInt64(nl_class & 63) + """Comptime: class c is the byte interval [rep_lo[c], rep_hi[c]], from + the first bytes `reps` (`_byte_classes`).""" + var nclasses = len(reps) + for ci in range(nclasses): + rep_lo[ci] = Int32(reps[ci]) + rep_hi[ci] = Int32(reps[ci + 1] - 1) if ci + 1 < nclasses else Int32( + 255 + ) + + +def _word_classes( + rep_lo: SIMD[DType.int32, 256], nclasses: Int +) -> SIMD[DType.uint64, 4]: + """Comptime: the classes of word bytes, as a class bitmask (classes are + pure w.r.t. the word set when the NFA has a word anchor).""" + var word_cls = SIMD[DType.uint64, 4](0) + for ci in range(nclasses): + if _is_word_byte(Int(rep_lo[ci])): + word_cls[ci >> 6] = word_cls[ci >> 6] | ( + UInt64(1) << UInt64(ci & 63) ) - _bs_set(consuming_bits, i) - elif kind == NFAStateKind.CHARSET: - var cs = nfa.states[i].charset_index - for r in range(len(nfa.charsets[cs].ranges)): - var lo = Int(nfa.charsets[cs].ranges[r].lo) - var hi = Int(nfa.charsets[cs].ranges[r].hi) - if lo > hi or lo > 255: - continue - if hi > 255: - hi = 255 - # _byte_classes marked lo and hi+1, so the classes of lo - # and hi bound exactly the classes inside [lo, hi]. - for ci in range(class_of[lo], class_of[hi] + 1): - cm[ci >> 6] = cm[ci >> 6] | (UInt64(1) << UInt64(ci & 63)) - if nfa.charsets[cs].negated: - # Classes are pure w.r.t. this charset, so negation is - # exact at class granularity. - for w in range(4): - cm[w] = ~cm[w] - for ci in range(nclasses, 256): - cm[ci >> 6] = cm[ci >> 6] & ~(UInt64(1) << UInt64(ci & 63)) - _bs_set(consuming_bits, i) - elif kind == NFAStateKind.MATCH: - _bs_set(match_bits, i) - elif kind == NFAStateKind.ANCHOR: - if at == AnchorKind.EOL or at == AnchorKind.EOL_MULTILINE: - _bs_set(eol_bits, i) - elif at == AnchorKind.BOL_MULTILINE: - has_bol_ml = True - cls_mask.append(cm) + return word_cls + + +def _eol_ok_bits( + nfa: NFA, + f: _FlatNFA, + mut eol_end_ok: _StateBits, + mut eol_nl_ok: _StateBits, +): + """Comptime: pending-EOL resolution per anchor state, as bitsets — does + its continuation reach MATCH at end of input (`eol_end_ok`) / at a + '\n' (`eol_nl_ok`)? The question `_check_eol_match` asks per member, + precomputed so a state's flags are bitset ANDs.""" + for s in range(len(f.kinds)): + if (f.eol_bits[s >> 6] >> UInt64(s & 63)) & 1 == 0: + continue + if _reaches_match(nfa, f.out1s[s], True): + _bs_set(eol_end_ok, s) + if f.anchors[s] == AnchorKind.EOL_MULTILINE and _reaches_match( + nfa, f.out1s[s], False + ): + _bs_set(eol_nl_ok, s) + + +def _bs_members(bits: _StateBits) -> List[Int]: + """Comptime: the members of a bitset, ascending.""" + var members = List[Int]() + for l in range(64): + var w = bits[l] + while w != 0: + members.append(64 * l + Int(count_trailing_zeros(w))) + w &= w - 1 + return members^ # --- Hopcroft minimization ------------------------------------------------- @@ -1256,44 +1338,21 @@ def build_eager_dfa(nfa: NFA, enabled: Bool, minimize: Bool = True) -> EagerDFA: if n > EDFA_NFA_CAP: return result^ # cannot bitset; would blow EDFA_STATE_CAP anyway - # --- Byte classes: intervals with a representative first byte. --- - var class_of = List[Int](fill=-1, length=256) - var reps = _byte_classes(nfa, class_of) - var nclasses = len(reps) + # --- Byte classes + one flat pass over the NFA (see _FlatNFA). --- + var flat = _FlatNFA(nfa) + var nclasses = flat.nclasses + var nl_class = flat.nl_class + ref kinds = flat.kinds + ref out1s = flat.out1s + ref out2s = flat.out2s + ref anchors = flat.anchors + ref cls_mask = flat.cls_mask + ref consuming_bits = flat.consuming_bits + ref match_bits = flat.match_bits + var has_bol_ml = flat.has_bol_ml var rep_lo = SIMD[DType.int32, 256](0) var rep_hi = SIMD[DType.int32, 256](0) - for ci in range(nclasses): - rep_lo[ci] = Int32(reps[ci]) - rep_hi[ci] = Int32(reps[ci + 1] - 1) if ci + 1 < nclasses else Int32( - 255 - ) - var nl_class = class_of[Int(CHAR_NEWLINE)] - - # --- One flat pass over the NFA (see _flatten_nfa). --- - var kinds = List[Int]() - var out1s = List[Int]() - var out2s = List[Int]() - var anchors = List[Int]() - var cls_mask = List[SIMD[DType.uint64, 4]]() - var consuming_bits = _StateBits(0) - var match_bits = _StateBits(0) - var eol_bits = _StateBits(0) - var has_bol_ml = False - _flatten_nfa( - nfa, - class_of, - nclasses, - nl_class, - kinds, - out1s, - out2s, - anchors, - cls_mask, - consuming_bits, - match_bits, - eol_bits, - has_bol_ml, - ) + _class_ranges(flat.reps, rep_lo, rep_hi) # --- Continuation closures, memoized by target state. --- # closure(union of targets) == union of closures, so per-member work in @@ -1330,21 +1389,9 @@ def build_eager_dfa(nfa: NFA, enabled: Bool, minimize: Bool = True) -> EagerDFA: var lazy_members = 0 var pool_threshold = 2 * n - # Pending-EOL resolution per anchor state, as bitsets: does the - # continuation reach MATCH at end of input / at a '\n'? (Same - # question `_check_eol_match` asks per member; precomputed so the - # per-state flag computation is two bitset ANDs.) var eol_end_ok = _StateBits(0) var eol_nl_ok = _StateBits(0) - for s in range(n): - if (eol_bits[s >> 6] >> UInt64(s & 63)) & 1 == 0: - continue - if _reaches_match(nfa, out1s[s], True): - _bs_set(eol_end_ok, s) - if anchors[s] == AnchorKind.EOL_MULTILINE and _reaches_match( - nfa, out1s[s], False - ): - _bs_set(eol_nl_ok, s) + _eol_ok_bits(nfa, flat, eol_end_ok, eol_nl_ok) # --- Word boundaries: look-behind class per state. --- # A DFA state is (set, prev_word): the set keeps word anchors as @@ -1356,12 +1403,7 @@ def build_eager_dfa(nfa: NFA, enabled: Bool, minimize: Bool = True) -> EagerDFA: # feed the word / non-word byte classes of the transitions. var has_wb = _nfa_has_word_anchor(nfa) var wb_bits = _word_anchor_bits(kinds, anchors) - var word_cls = SIMD[DType.uint64, 4](0) # classes of word bytes - for ci in range(nclasses): - if _is_word_byte(Int(rep_lo[ci])): - word_cls[ci >> 6] = word_cls[ci >> 6] | ( - UInt64(1) << UInt64(ci & 63) - ) + var word_cls = _word_classes(rep_lo, nclasses) # --- State-set bookkeeping: bitsets, hashes in SIMD lanes, flags. --- var sets_bits = List[_StateBits]() diff --git a/emberregex/static_lfdfa.mojo b/emberregex/static_lfdfa.mojo index 3b9b890..dd0c84e 100644 --- a/emberregex/static_lfdfa.mojo +++ b/emberregex/static_lfdfa.mojo @@ -49,8 +49,6 @@ from std.collections import Array from std.math import iota from .ast import AnchorKind -from .constants import CHAR_NEWLINE -from .dfa import _reaches_match from .nfa import NFA, NFAStateKind from .static_dfa import ( EDFA_EOL_AT_END, @@ -64,16 +62,17 @@ from .static_dfa import ( _StateBits, _WB_PREV_SALT, _bs_set, - _is_word_byte, + _FlatNFA, + _class_ranges, + _eol_ok_bits, _lane_word, - _byte_classes, _edfa_finish, - _flatten_nfa, _nfa_has_word_anchor, _wb_anchor_flags, _wb_holds, _wb_normalize, _word_anchor_bits, + _word_classes, WB_PENDING, WB_RESOLVE, edfa_walk_from, @@ -289,58 +288,28 @@ def build_lf_dfa( return result^ # --- Byte classes + flat NFA views (shared with build_eager_dfa). --- - var class_of = List[Int](fill=-1, length=256) - var reps = _byte_classes(nfa, class_of) - var nclasses = len(reps) + var flat = _FlatNFA(nfa) + var nclasses = flat.nclasses + var nl_class = flat.nl_class + var nwords = (nclasses + 63) >> 6 + ref kinds = flat.kinds + ref out1s = flat.out1s + ref out2s = flat.out2s + ref anchors = flat.anchors + ref cls_mask = flat.cls_mask + ref consuming_bits = flat.consuming_bits + ref match_bits = flat.match_bits + ref eol_bits = flat.eol_bits + var has_bol_ml = flat.has_bol_ml var rep_lo = SIMD[DType.int32, 256](0) var rep_hi = SIMD[DType.int32, 256](0) - for ci in range(nclasses): - rep_lo[ci] = Int32(reps[ci]) - rep_hi[ci] = Int32(reps[ci + 1] - 1) if ci + 1 < nclasses else Int32( - 255 - ) - var nl_class = class_of[Int(CHAR_NEWLINE)] - var nwords = (nclasses + 63) >> 6 + _class_ranges(flat.reps, rep_lo, rep_hi) - var kinds = List[Int]() - var out1s = List[Int]() - var out2s = List[Int]() - var anchors = List[Int]() - var cls_mask = List[SIMD[DType.uint64, 4]]() - var consuming_bits = _StateBits(0) - var match_bits = _StateBits(0) - var eol_bits = _StateBits(0) - var has_bol_ml = False - _flatten_nfa( - nfa, - class_of, - nclasses, - nl_class, - kinds, - out1s, - out2s, - anchors, - cls_mask, - consuming_bits, - match_bits, - eol_bits, - has_bol_ml, - ) - - # Pending-EOL resolution, per anchor state: does its continuation - # reach MATCH at end of input / at a '\n'? (The continuation never - # consumes on this lane — see _eol_ml_continuation_consumes.) + # Pending-EOL resolution (the continuation never consumes on this + # lane — see _eol_ml_continuation_consumes). var eol_end_ok = _StateBits(0) var eol_nl_ok = _StateBits(0) - for s in range(n): - if (eol_bits[s >> 6] >> UInt64(s & 63)) & 1 == 0: - continue - if _reaches_match(nfa, out1s[s], True): - _bs_set(eol_end_ok, s) - if anchors[s] == AnchorKind.EOL_MULTILINE and _reaches_match( - nfa, out1s[s], False - ): - _bs_set(eol_nl_ok, s) + _eol_ok_bits(nfa, flat, eol_end_ok, eol_nl_ok) # --- Ordered closures, memoized per (target, context) as chunks. --- var pool = List[Int]() @@ -361,12 +330,7 @@ def build_lf_dfa( # EOL_MULTILINE resolving at '\n'). var has_wb = _nfa_has_word_anchor(nfa) var wb_bits = _word_anchor_bits(kinds, anchors) - var word_cls = SIMD[DType.uint64, 4](0) # classes of word bytes - for ci in range(nclasses): - if _is_word_byte(Int(rep_lo[ci])): - word_cls[ci >> 6] = word_cls[ci >> 6] | ( - UInt64(1) << UInt64(ci & 63) - ) + var word_cls = _word_classes(rep_lo, nclasses) # classes of word bytes # The restart closures (mid-line / after '\n') as lane vectors, with # their flag bytes, whether they end in MATCH, and their pending word diff --git a/emberregex/static_rdfa.mojo b/emberregex/static_rdfa.mojo index 8031979..00d4a4c 100644 --- a/emberregex/static_rdfa.mojo +++ b/emberregex/static_rdfa.mojo @@ -74,9 +74,9 @@ from .static_dfa import ( _bs_eq, _bs_hash, _bs_set, - _byte_classes, - _flatten_nfa, - _is_word_byte, + _FlatNFA, + _class_ranges, + _word_classes, _minimize, _nfa_has_word_anchor, _wb_holds, @@ -287,42 +287,17 @@ def build_reverse_dfa(nfa: NFA, enabled: Bool) -> RDFA: return result^ var preds = _reverse_edges(nfa) - var class_of = List[Int](fill=-1, length=256) - var reps = _byte_classes(nfa, class_of) - var nclasses = len(reps) + var flat = _FlatNFA(nfa) + var nclasses = flat.nclasses + var nl_class = flat.nl_class + ref kinds = flat.kinds + ref anchors = flat.anchors + ref cls_mask = flat.cls_mask + ref eol_bits = flat.eol_bits + var has_bol_ml = flat.has_bol_ml var rep_lo = SIMD[DType.int32, 256](0) var rep_hi = SIMD[DType.int32, 256](0) - for ci in range(nclasses): - rep_lo[ci] = Int32(reps[ci]) - rep_hi[ci] = Int32(reps[ci + 1] - 1) if ci + 1 < nclasses else Int32( - 255 - ) - var nl_class = class_of[Int(CHAR_NEWLINE)] - - var kinds = List[Int]() - var out1s = List[Int]() - var out2s = List[Int]() - var anchors = List[Int]() - var cls_mask = List[SIMD[DType.uint64, 4]]() - var consuming_bits = _StateBits(0) - var match_bits = _StateBits(0) - var eol_bits = _StateBits(0) - var has_bol_ml = False - _flatten_nfa( - nfa, - class_of, - nclasses, - nl_class, - kinds, - out1s, - out2s, - anchors, - cls_mask, - consuming_bits, - match_bits, - eol_bits, - has_bol_ml, - ) + _class_ranges(flat.reps, rep_lo, rep_hi) var pred_data = List[Int]() var pred_off = List[Int]() @@ -426,12 +401,7 @@ def build_reverse_dfa(nfa: NFA, enabled: Bool) -> RDFA: var gval_n = List[_StateBits]() var one_seed = List[Int](fill=0, length=1) var need_nl_variant = has_bol_ml or _bs_any(eol_bits) - var word_cls = SIMD[DType.uint64, 4](0) - for ci in range(nclasses): - if _is_word_byte(Int(rep_lo[ci])): - word_cls[ci >> 6] = word_cls[ci >> 6] | ( - UInt64(1) << UInt64(ci & 63) - ) + var word_cls = _word_classes(rep_lo, nclasses) var rows = List[SIMD[DType.int32, 256]]() var cur = 0 diff --git a/test/test_eager_dfa.mojo b/test/test_eager_dfa.mojo index f5b66e9..99f284d 100644 --- a/test/test_eager_dfa.mojo +++ b/test/test_eager_dfa.mojo @@ -7,7 +7,6 @@ regression can't hide behind the identical-semantics lazy path. from emberregex import Regex from emberregex.simd_kernels import HAS_FAST_BYTE_SHUFFLE -from emberregex.constants import CHAR_NEWLINE from emberregex.engine import _build_static_nfa from emberregex.ast import AnchorKind from emberregex.nfa import NFA, NFAStateKind @@ -16,16 +15,15 @@ from emberregex.static_dfa import ( EDFA_DEAD, EDFA_STATE_CAP, EagerDFA, + _FlatNFA, WB_PENDING, WB_RESOLVE, _MIN_CAP, _StateBits, _bs_eq, - _byte_classes, _closure_pool, _edfa_has_accel, _flat_closure, - _flatten_nfa, _minimize, build_eager_dfa, edfa_id_dtype, @@ -617,34 +615,11 @@ def test_differential_table_walk_big_merge() raises: def _pool_parity_impl(nfa: NFA) -> Bool: """Comptime: `_closure_pool` agrees with a per-target `_flat_closure` DFS for EVERY state, in both `after_newline` variants.""" - var class_of = List[Int](fill=-1, length=256) - var reps = _byte_classes(nfa, class_of) - var nclasses = len(reps) - var nl_class = class_of[Int(CHAR_NEWLINE)] - var kinds = List[Int]() - var out1s = List[Int]() - var out2s = List[Int]() - var anchors = List[Int]() - var cls_mask = List[SIMD[DType.uint64, 4]]() - var consuming_bits = _StateBits(0) - var match_bits = _StateBits(0) - var eol_bits = _StateBits(0) - var has_bol_ml = False - _flatten_nfa( - nfa, - class_of, - nclasses, - nl_class, - kinds, - out1s, - out2s, - anchors, - cls_mask, - consuming_bits, - match_bits, - eol_bits, - has_bol_ml, - ) + var flat = _FlatNFA(nfa) + ref kinds = flat.kinds + ref out1s = flat.out1s + ref out2s = flat.out2s + ref anchors = flat.anchors var n = len(kinds) var pool_o = _closure_pool(kinds, out1s, out2s, anchors, False) var pool_n = _closure_pool(kinds, out1s, out2s, anchors, True) @@ -708,40 +683,6 @@ def test_accel_run_to_end_of_input_on_the_table_walk() raises: assert_false(re.match(String("x") + String("q") * 69 + "a").matched) -def _flat_views( - nfa: NFA, - mut kinds: List[Int], - mut out1s: List[Int], - mut out2s: List[Int], - mut anchors: List[Int], -): - """Runtime: the flat NFA views `_flat_closure` walks.""" - var class_of = List[Int](fill=-1, length=256) - var reps = _byte_classes(nfa, class_of) - var nclasses = len(reps) - var nl_class = class_of[Int(CHAR_NEWLINE)] - var cls_mask = List[SIMD[DType.uint64, 4]]() - var consuming_bits = _StateBits(0) - var match_bits = _StateBits(0) - var eol_bits = _StateBits(0) - var has_bol_ml = False - _flatten_nfa( - nfa, - class_of, - nclasses, - nl_class, - kinds, - out1s, - out2s, - anchors, - cls_mask, - consuming_bits, - match_bits, - eol_bits, - has_bol_ml, - ) - - def _bs_has(b: _StateBits, i: Int) -> Bool: return (b[i >> 6] >> UInt64(i & 63)) & 1 != 0 @@ -752,11 +693,11 @@ def _wb_closure( """Runtime: the closure seeded at `p`'s first word anchor, as (the anchor is a member, its continuation is a member).""" var nfa = _build_static_nfa(p) - var kinds = List[Int]() - var out1s = List[Int]() - var out2s = List[Int]() - var anchors = List[Int]() - _flat_views(nfa, kinds, out1s, out2s, anchors) + var flat = _FlatNFA(nfa) + ref kinds = flat.kinds + ref out1s = flat.out1s + ref out2s = flat.out2s + ref anchors = flat.anchors var a = -1 for s in range(len(kinds)): if kinds[s] == NFAStateKind.ANCHOR and ( @@ -803,11 +744,11 @@ def test_flat_closure_saves_and_out_of_range_seed_at_runtime() raises: # A SAVE expands like an epsilon (the group's CHAR lands in the set); # a seed outside the state range yields the empty set. var nfa = _build_static_nfa("(a)b") - var kinds = List[Int]() - var out1s = List[Int]() - var out2s = List[Int]() - var anchors = List[Int]() - _flat_views(nfa, kinds, out1s, out2s, anchors) + var flat = _FlatNFA(nfa) + ref kinds = flat.kinds + ref out1s = flat.out1s + ref out2s = flat.out2s + ref anchors = flat.anchors var start = nfa.start assert_equal(kinds[start], NFAStateKind.SAVE) var bits = _flat_closure(kinds, out1s, out2s, anchors, start, True, True) From 25a18978132392c69c33778b02d47819a3d8459c Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:46:42 -0600 Subject: [PATCH 47/65] cleanup: fix stale docstrings (_edfa_finish extra starts, PROBE_RANKS) Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/optimize.mojo | 4 ++-- emberregex/static_dfa.mojo | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/emberregex/optimize.mojo b/emberregex/optimize.mojo index 2ab71d9..e7adaa2 100644 --- a/emberregex/optimize.mojo +++ b/emberregex/optimize.mojo @@ -828,7 +828,7 @@ def extract_inner_literal(nfa: NFA, cyclic: List[Bool]) -> InnerLiteral: Among the collected runs, positions at fixed offset 0 are dropped and the rarest run of length >= 2 wins (score = the run's rarest byte by - _probe_rank_table, caseless positions counting both cases; ties + PROBE_RANKS, caseless positions counting both cases; ties prefer the longer run).""" var res = InnerLiteral() var n = len(nfa.states) @@ -1056,7 +1056,7 @@ def select_probe_offsets( prefix: List[UInt8], caseless: List[Bool] ) -> Tuple[Int, Int]: """Comptime: offsets of the two rarest prefix positions for the - two-byte candidate filter, per _probe_rank_table. A caseless position + two-byte candidate filter, per PROBE_RANKS. A caseless position matches both cases, so its rank is the sum of both cases' frequencies. Ties prefer later offsets (larger spread rejects repeated-byte runs sooner). Requires len(prefix) >= 2; returns (off_a, off_b) with diff --git a/emberregex/static_dfa.mojo b/emberregex/static_dfa.mojo index 02bd6c4..bbb2284 100644 --- a/emberregex/static_dfa.mojo +++ b/emberregex/static_dfa.mojo @@ -827,7 +827,7 @@ def _byte_classes(nfa: NFA, mut class_of: List[Int]) -> List[Int]: struct _FlatNFA: """Byte classes plus one flat pass over the NFA — the setup every bitset - determinizer shares. Built ONCE per build, in place (`var f = + determinizer shares. Built ONCE per build, in place (`var flat = _FlatNFA(nfa)`); callers bind the fields they need to locals with `ref` and pass those, never the struct, into the hot loops, which then never carry an aggregate across a call boundary.""" @@ -960,7 +960,7 @@ def _eol_ok_bits( ): """Comptime: pending-EOL resolution per anchor state, as bitsets — does its continuation reach MATCH at end of input (`eol_end_ok`) / at a - '\n' (`eol_nl_ok`)? The question `_check_eol_match` asks per member, + '\\n' (`eol_nl_ok`)? The question `_check_eol_match` asks per member, precomputed so a state's flags are bitset ANDs.""" for s in range(len(f.kinds)): if (f.eol_bits[s >> 6] >> UInt64(s & 63)) & 1 == 0: @@ -1675,8 +1675,8 @@ def _edfa_finish( `starts` holds any number of start ids in the caller's order, the first three being (other, after-'\\n', at-0) for the `start_*` fields; the permuted ids come back in the same order so a producer - with extra start contexts (the leftmost-first DFA's anchored starts) - can record them. The first `nregion` of them are the candidates for + with extra ids to track (the leftmost-first DFA's look-behind-"word" + states, `LFDFA.prev_ids`) can record them. The first `nregion` of them are the candidates for region acceleration (the unanchored start contexts). Marks `result` valid. """ From 3c888edb0e47a857b2531c864d2814a17b5ebe17 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 12:55:37 -0600 Subject: [PATCH 48/65] cleanup: mojo format (pre-existing drift in engine.mojo slots=Array calls) Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/engine.mojo | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/emberregex/engine.mojo b/emberregex/engine.mojo index 77ee782..59b4588 100644 --- a/emberregex/engine.mojo +++ b/emberregex/engine.mojo @@ -2337,9 +2337,7 @@ struct Regex[pattern: String](Copyable, Movable): end=self._lf_end_at( input_bytes, pos, match_end ), - slots=Array[Int, Self._num_slots]( - fill=-1 - ), + slots=Array[Int, Self._num_slots](fill=-1), ) var nl = simd_find_byte(input_bytes, CHAR_NEWLINE, pos) if nl < 0: @@ -2364,9 +2362,7 @@ struct Regex[pattern: String](Copyable, Movable): end=self._lf_end_at( input_bytes, pos, match_end ), - slots=Array[Int, Self._num_slots]( - fill=-1 - ), + slots=Array[Int, Self._num_slots](fill=-1), ) pos = _scan_bump[Self._is_unicode](input_bytes, pos) else: @@ -2380,9 +2376,7 @@ struct Regex[pattern: String](Copyable, Movable): end=self._lf_end_at( input_bytes, range[0], range[1] ), - slots=Array[Int, Self._num_slots]( - fill=-1 - ), + slots=Array[Int, Self._num_slots](fill=-1), ) return MatchResult[Self._num_slots].no_match() return MatchResult[Self._num_slots].no_match() From 9bfa043ccf84f8fe5aded37e54d2a355222e6cc2 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 14:56:49 -0600 Subject: [PATCH 49/65] cleanup: one set Pike scan with a comptime som flag set_pike_scan and set_pike_som_scan share _pike_scan[som]. The SOM lane's per-thread / per-report starts move out of the recursive closure: every state one closure adds carries the caller's start, so the parallel lists are resized after _set_add_state returns. The plain instantiation keeps the unchanged _set_add_state and _flush_reports (asm-diffed against the base: identical save one cmp/b.ne -> eor/tbnz pair). Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/set_pike.mojo | 406 +++++++++++++++------------------------ 1 file changed, 159 insertions(+), 247 deletions(-) diff --git a/emberregex/set_pike.mojo b/emberregex/set_pike.mojo index d4fa91c..54fa08a 100644 --- a/emberregex/set_pike.mojo +++ b/emberregex/set_pike.mojo @@ -67,207 +67,6 @@ struct SetSpan(Equatable, TrivialRegisterPassable, Writable): ) -def _som_add_state[ - origin: Origin, // -]( - nfa: NFA, - mut state_list: List[Int], - mut state_som: List[Int], - mut gen: List[Int], - gen_val: Int, - start_idx: Int, - som: Int, - input: Span[Byte, origin], - input_len: Int, - pos: Int, - mut ids: List[Int], - mut id_som: List[Int], -): - """`_set_add_state` carrying a start offset per thread. - - Threads are expanded in nondecreasing `som` order — the caller walks - the previous list in order and re-seeds last — so the generation - counter, which lets the FIRST claimant of a state win, hands every - state its leftmost reachable start. - """ - var idx = start_idx - var num_st = len(nfa.states) - while True: - if idx < 0 or idx >= num_st: - return - if gen.unsafe_get(idx) == gen_val: - return - gen.unsafe_set(idx, gen_val) - - ref state = nfa.states.unsafe_get(idx) - var kind = state.kind - - if kind == NFAStateKind.SPLIT: - _som_add_state( - nfa, - state_list, - state_som, - gen, - gen_val, - state.out1, - som, - input, - input_len, - pos, - ids, - id_som, - ) - idx = state.out2 - elif kind == NFAStateKind.SAVE: - idx = state.out1 - elif kind == NFAStateKind.ANCHOR: - if _bt_check_anchor(state.anchor_type, input, input_len, pos): - idx = state.out1 - else: - return - elif kind == NFAStateKind.MATCH: - ids.append(state.report_id) - id_som.append(som) - return - else: - state_list.append(idx) - state_som.append(som) - return - - -def set_pike_som_scan[ - origin: Origin, // -](nfa: NFA, input: Span[Byte, origin]) -> List[SetSpan]: - """All-ends scan carrying start-of-match, for sets the reverse DFA - cannot build (word boundaries, cap blowups). - - Same contract as `set_pike_scan` plus a leftmost `start` per report. - """ - var out = List[SetSpan]() - var num_states = len(nfa.states) - if num_states == 0: - return out^ - var input_len = len(input) - - var gen = List[Int](length=num_states, fill=0) - var gen_counter = 0 - var current = List[Int](capacity=num_states) - var current_som = List[Int](capacity=num_states) - var next_states = List[Int](capacity=num_states) - var next_som = List[Int](capacity=num_states) - var ids = List[Int]() - var id_som = List[Int]() - - gen_counter += 1 - _som_add_state( - nfa, - current, - current_som, - gen, - gen_counter, - nfa.start, - 0, - input, - input_len, - 0, - ids, - id_som, - ) - _flush_spans(ids, id_som, 0, out) - - var pos = 0 - while pos < input_len: - var ch = UInt32(input.unsafe_get(pos)) - gen_counter += 1 - next_states.clear() - next_som.clear() - ids.clear() - id_som.clear() - - for i in range(len(current)): - var s = current.unsafe_get(i) - var som = current_som.unsafe_get(i) - ref state = nfa.states.unsafe_get(s) - var kind = state.kind - var ok: Bool - if kind == NFAStateKind.CHAR: - ok = ch == state.char_value - elif kind == NFAStateKind.ANY: - ok = ch != UInt32(CHAR_NEWLINE) - elif kind == NFAStateKind.CHARSET: - ok = nfa.charsets.unsafe_get(state.charset_index).contains(ch) - else: - ok = False - if ok: - _som_add_state( - nfa, - next_states, - next_som, - gen, - gen_counter, - state.out1, - som, - input, - input_len, - pos + 1, - ids, - id_som, - ) - - _som_add_state( - nfa, - next_states, - next_som, - gen, - gen_counter, - nfa.start, - pos + 1, - input, - input_len, - pos + 1, - ids, - id_som, - ) - - var tmp = current^ - current = next_states^ - next_states = tmp^ - var tmp2 = current_som^ - current_som = next_som^ - next_som = tmp2^ - pos += 1 - if _mid_codepoint(nfa, input, pos): - var kept = List[Int]() - var kept_som = List[Int]() - for i in range(len(ids)): - if _keep_id_at(nfa, ids[i]): - kept.append(ids[i]) - kept_som.append(id_som[i]) - ids = kept^ - id_som = kept_som^ - _flush_spans(ids, id_som, pos, out) - - return out^ - - -def _flush_spans( - mut ids: List[Int], mut som: List[Int], end: Int, mut out: List[SetSpan] -): - """Emit this position's reports in ascending id order.""" - for i in range(1, len(ids)): - var key = ids[i] - var key_som = som[i] - var j = i - 1 - while j >= 0 and ids[j] > key: - ids[j + 1] = ids[j] - som[j + 1] = som[j] - j -= 1 - ids[j + 1] = key - som[j + 1] = key_som - for i in range(len(ids)): - out.append(SetSpan(ids[i], som[i], end)) - - def _set_add_state[ origin: Origin, // ]( @@ -388,6 +187,56 @@ def _keep_id_at(nfa: NFA, id: Int) -> Bool: return id >= len(nfa.pattern_unicode) or not nfa.pattern_unicode[id] +comptime _PikeOut[som: Bool] = SetSpan if som else SetMatch + + +@always_inline +def _pike_add[ + origin: Origin, //, som: Bool +]( + nfa: NFA, + mut state_list: List[Int], + mut state_som: List[Int], + mut gen: List[Int], + gen_val: Int, + start_idx: Int, + start: Int, + input: Span[Byte, origin], + input_len: Int, + pos: Int, + mut ids: List[Int], + mut id_som: List[Int], +): + """`_set_add_state`, plus (SOM lane only) the start offset of every + thread and report it added: one closure carries one start throughout, + so the parallel lists just extend to their partners' lengths. The + plain lane never touches them.""" + _set_add_state( + nfa, state_list, gen, gen_val, start_idx, input, input_len, pos, ids + ) + comptime if som: + state_som.resize(len(state_list), start) + id_som.resize(len(ids), start) + + +def _flush_spans( + mut ids: List[Int], mut som: List[Int], end: Int, mut out: List[SetSpan] +): + """Emit this position's reports in ascending id order.""" + for i in range(1, len(ids)): + var key = ids[i] + var key_som = som[i] + var j = i - 1 + while j >= 0 and ids[j] > key: + ids[j + 1] = ids[j] + som[j + 1] = som[j] + j -= 1 + ids[j + 1] = key + som[j + 1] = key_som + for i in range(len(ids)): + out.append(SetSpan(ids[i], som[i], end)) + + def _flush_reports(mut ids: List[Int], end: Int, mut out: List[SetMatch]): """Emit this position's reports in ascending id order.""" for i in range(1, len(ids)): @@ -401,14 +250,37 @@ def _flush_reports(mut ids: List[Int], end: Int, mut out: List[SetMatch]): out.append(SetMatch(ids[i], end)) -def set_pike_scan[ - origin: Origin, // -](nfa: NFA, input: Span[Byte, origin]) -> List[SetMatch]: - """Scan the whole input, reporting every (id, end) per the contract. +@always_inline +def _flush[ + som: Bool +]( + mut ids: List[Int], + mut id_som: List[Int], + end: Int, + mut out: List[_PikeOut[som]], +): + comptime if som: + _flush_spans(ids, id_som, end, rebind[List[SetSpan]](out)) + else: + _flush_reports(ids, end, rebind[List[SetMatch]](out)) + + +def _pike_scan[ + origin: Origin, //, som: Bool +](nfa: NFA, input: Span[Byte, origin]) -> List[_PikeOut[som]]: + """The all-match scan behind `set_pike_scan` (som=False) and + `set_pike_som_scan` (som=True). The SOM lane carries a start per + thread and per report in lists parallel to `current` / `ids`; the + plain lane instantiates none of that bookkeeping. + + Threads are expanded in nondecreasing start order — the previous list + is walked in order and the start closure re-seeded last — so the + generation counter, which lets the FIRST claimant of a state win, + hands every state its leftmost reachable start. Non-mutating: all working buffers are local to the call. """ - var out = List[SetMatch]() + var out = List[_PikeOut[som]]() var num_states = len(nfa.states) if num_states == 0: return out^ @@ -419,12 +291,29 @@ def set_pike_scan[ var current = List[Int](capacity=num_states) var next_states = List[Int](capacity=num_states) var ids = List[Int]() + var current_som = List[Int]() + var next_som = List[Int]() + var id_som = List[Int]() + comptime if som: + current_som.reserve(num_states) + next_som.reserve(num_states) gen_counter += 1 - _set_add_state( - nfa, current, gen, gen_counter, nfa.start, input, input_len, 0, ids + _pike_add[som]( + nfa, + current, + current_som, + gen, + gen_counter, + nfa.start, + 0, + input, + input_len, + 0, + ids, + id_som, ) - _flush_reports(ids, 0, out) + _flush[som](ids, id_som, 0, out) var pos = 0 while pos < input_len: @@ -432,75 +321,98 @@ def set_pike_scan[ gen_counter += 1 next_states.clear() ids.clear() + comptime if som: + next_som.clear() + id_som.clear() for i in range(len(current)): var s = current.unsafe_get(i) ref state = nfa.states.unsafe_get(s) var kind = state.kind + var ok: Bool if kind == NFAStateKind.CHAR: - if ch == state.char_value: - _set_add_state( - nfa, - next_states, - gen, - gen_counter, - state.out1, - input, - input_len, - pos + 1, - ids, - ) + ok = ch == state.char_value elif kind == NFAStateKind.ANY: - if ch != UInt32(CHAR_NEWLINE): - _set_add_state( - nfa, - next_states, - gen, - gen_counter, - state.out1, - input, - input_len, - pos + 1, - ids, - ) + ok = ch != UInt32(CHAR_NEWLINE) elif kind == NFAStateKind.CHARSET: - if nfa.charsets.unsafe_get(state.charset_index).contains(ch): - _set_add_state( - nfa, - next_states, - gen, - gen_counter, - state.out1, - input, - input_len, - pos + 1, - ids, - ) + ok = nfa.charsets.unsafe_get(state.charset_index).contains(ch) + else: + ok = False + if ok: + var start = 0 + comptime if som: + start = current_som.unsafe_get(i) + _pike_add[som]( + nfa, + next_states, + next_som, + gen, + gen_counter, + state.out1, + start, + input, + input_len, + pos + 1, + ids, + id_som, + ) # All-match, unanchored: re-seed the start closure at the next # position so matches may begin anywhere. - _set_add_state( + _pike_add[som]( nfa, next_states, + next_som, gen, gen_counter, nfa.start, + pos + 1, input, input_len, pos + 1, ids, + id_som, ) var tmp = current^ current = next_states^ next_states = tmp^ + comptime if som: + var tmp2 = current_som^ + current_som = next_som^ + next_som = tmp2^ pos += 1 if _mid_codepoint(nfa, input, pos): var kept = List[Int]() + var kept_som = List[Int]() for i in range(len(ids)): if _keep_id_at(nfa, ids[i]): kept.append(ids[i]) + comptime if som: + kept_som.append(id_som[i]) ids = kept^ - _flush_reports(ids, pos, out) + comptime if som: + id_som = kept_som^ + _flush[som](ids, id_som, pos, out) return out^ + + +@always_inline +def set_pike_scan[ + origin: Origin, // +](nfa: NFA, input: Span[Byte, origin]) -> List[SetMatch]: + """Scan the whole input, reporting every (id, end) per the contract.""" + return _pike_scan[som=False](nfa, input) + + +@always_inline +def set_pike_som_scan[ + origin: Origin, // +](nfa: NFA, input: Span[Byte, origin]) -> List[SetSpan]: + """All-ends scan carrying start-of-match, for sets the reverse DFA + cannot build (word boundaries, cap blowups). + + Same contract as `set_pike_scan` plus a leftmost `start` per report. + """ + return _pike_scan[som=True](nfa, input) From 1ba8c47ce533d0a37bf238035e737b4fd762229f Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 15:06:23 -0600 Subject: [PATCH 50/65] cleanup: share the Teddy front end between litset and Rose Rose's _rose_pos_masks re-derived _litset_pos_masks from the flat meta pools and rose_scan copied litset_scan's chunk loop. The masks now come from litset_masks(Self._rose.lit) at decl level, and both scans run teddy_front_end with their verify step as an inlined closure over an immutable view of the input. The lane probe's teddy/rose functions and constant pools are asm-identical to the previous build. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/set_engine.mojo | 4 +- emberregex/set_literal.mojo | 100 +++++++++++++++++++++++++----------- emberregex/set_rose.mojo | 96 +++++----------------------------- 3 files changed, 86 insertions(+), 114 deletions(-) diff --git a/emberregex/set_engine.mojo b/emberregex/set_engine.mojo index f1be3ee..5141f37 100644 --- a/emberregex/set_engine.mojo +++ b/emberregex/set_engine.mojo @@ -68,7 +68,7 @@ from .set_dfa import ( mdfa_slices_arr, mdfa_table_str, ) -from .set_literal import extract_literal_set, litset_scan +from .set_literal import extract_literal_set, litset_masks, litset_scan from .set_nfa import build_union_nfa, build_union_subset_nfa, _union_any_unicode from .set_pike import utf8_mid_codepoint from .set_pike import ( @@ -284,6 +284,7 @@ struct RegexSet[ ) comptime _ROSE_BCLS = rose_bcls_arr[rose_bcls_len(Self._rose)](Self._rose) comptime _ROSE_LOOK = rose_look_arr[rose_look_len(Self._rose)](Self._rose) + comptime _ROSE_MASKS = litset_masks(Self._rose.lit) # Patterns Rose could not decompose keep a per-byte automaton, over # their own (smaller, better-accelerating) union. @@ -749,6 +750,7 @@ struct RegexSet[ lits=Self._ROSE_LITS, bcls=Self._ROSE_BCLS, look=Self._ROSE_LOOK, + masks=Self._ROSE_MASKS, ](input) comptime if Self._has_residual: return merge_reports(reports^, self._scan_residual(input)) diff --git a/emberregex/set_literal.mojo b/emberregex/set_literal.mojo index bfd7eef..313c655 100644 --- a/emberregex/set_literal.mojo +++ b/emberregex/set_literal.mojo @@ -307,6 +307,70 @@ def _litset_pos_masks( return (lo, hi) +comptime _PosMasks = Tuple[_NibbleTable, _NibbleTable] +comptime TeddyMasks = Tuple[_PosMasks, _PosMasks, _PosMasks] + + +def litset_masks(ls: LiteralSet) -> TeddyMasks: + """Comptime: the nibble tables for the first k = min(3, min_len) + literal positions (positions past k repeat position 0; the front end + never reads them).""" + var k = min(3, ls.min_len) + return ( + _litset_pos_masks(ls, 0), + _litset_pos_masks(ls, 1 if k > 1 else 0), + _litset_pos_masks(ls, 2 if k > 2 else 0), + ) + + +@always_inline +def teddy_front_end[ + origin: ImmOrigin, + F: def(Int, UInt8) -> None, + //, + min_len: Int, + masks: TeddyMasks, +](input: Span[Byte, origin], verify: F): + """The bucketed-Teddy candidate loop shared by `litset_scan` and + `rose_scan`: `verify(at, bucket_mask)` for every candidate lane, then + `verify(at, 0xFF)` at each tail position with room for the shortest + literal. `input` is an immutable view so the verify closure can + capture it too.""" + comptime W = simd_width_of[DType.uint8]() + comptime k = min(3, min_len) + comptime m0 = masks[0] + comptime m1 = masks[1] + comptime m2 = masks[2] + + var input_len = len(input) + var pos = 0 + var ptr = Pointer(input.unsafe_ptr()) + + while pos + W <= input_len: + var v = ptr.unsafe_offset(pos).unsafe_load[width=W]() + var lo = v & 0x0F + var hi = v >> 4 + var cand = nibble_lookup(m0[0], lo) & nibble_lookup(m0[1], hi) + comptime if k > 1: + var c1 = nibble_lookup(m1[0], lo) & nibble_lookup(m1[1], hi) + cand &= c1.shift_left[1]() + comptime if k > 2: + var c2 = nibble_lookup(m2[0], lo) & nibble_lookup(m2[1], hi) + cand &= c2.shift_left[2]() + var bits = lane_bits(cand.ne(0)) + while bits != 0: + var lane = first_lane_index(bits) + verify(pos + lane, cand[lane]) + bits = clear_first_lane(bits) + # The last k-1 lanes were masked off by the zero-filling lane + # shifts; rescan them as the head of the next chunk. + pos += W - (k - 1) + + while pos + min_len <= input_len: + verify(pos, UInt8(0xFF)) + pos += 1 + + @always_inline def _litset_verify_at[ origin: Origin, //, ls: LiteralSet @@ -358,40 +422,14 @@ def litset_scan[ ](input: Span[Byte, origin]) -> List[SetMatch]: """Scan the whole input, reporting every (id, end) per the set contract. Non-mutating; buffers are local.""" - comptime W = simd_width_of[DType.uint8]() - comptime k = min(3, ls.min_len) - comptime m0 = _litset_pos_masks(ls, 0) - comptime m1 = _litset_pos_masks(ls, 1 if k > 1 else 0) - comptime m2 = _litset_pos_masks(ls, 2 if k > 2 else 0) - var out = List[SetMatch]() - var input_len = len(input) - var pos = 0 - var ptr = Pointer(input.unsafe_ptr()) + var inp = Span[Byte, ImmOrigin(origin)](input) - while pos + W <= input_len: - var v = ptr.unsafe_offset(pos).unsafe_load[width=W]() - var lo = v & 0x0F - var hi = v >> 4 - var cand = nibble_lookup(m0[0], lo) & nibble_lookup(m0[1], hi) - comptime if k > 1: - var c1 = nibble_lookup(m1[0], lo) & nibble_lookup(m1[1], hi) - cand &= c1.shift_left[1]() - comptime if k > 2: - var c2 = nibble_lookup(m2[0], lo) & nibble_lookup(m2[1], hi) - cand &= c2.shift_left[2]() - var bits = lane_bits(cand.ne(0)) - while bits != 0: - var lane = first_lane_index(bits) - _litset_verify_at[ls=ls](input, pos + lane, cand[lane], out) - bits = clear_first_lane(bits) - # The last k-1 lanes were masked off by the zero-filling lane - # shifts; rescan them as the head of the next chunk. - pos += W - (k - 1) + @always_inline + def verify(at: Int, bucket_mask: UInt8) {mut out, imm inp}: + _litset_verify_at[ls=ls](inp, at, bucket_mask, out) - while pos + ls.min_len <= input_len: - _litset_verify_at[ls=ls](input, pos, UInt8(0xFF), out) - pos += 1 + teddy_front_end[min_len=ls.min_len, masks=litset_masks(ls)](inp, verify) _sort_reports(out) # Collapse duplicate (id, end) pairs: same-id arms of an in-pattern diff --git a/emberregex/set_rose.mojo b/emberregex/set_rose.mojo index 1940028..1c736e1 100644 --- a/emberregex/set_rose.mojo +++ b/emberregex/set_rose.mojo @@ -72,8 +72,6 @@ sort+dedup. """ from std.collections import Array -from std.math import min -from std.sys import simd_width_of from .charset import BITMAP_WIDTH from .constants import CHAR_NEWLINE @@ -83,8 +81,10 @@ from .parser import parse from .set_literal import ( LITSET_MAX, LiteralSet, + TeddyMasks, _NUM_BUCKETS, _assign_buckets, + teddy_front_end, ) from .set_pike import SetMatch from .set_semantics import ( @@ -92,12 +92,7 @@ from .set_semantics import ( EXT_HAMMING_DISTANCE, ext_of, ) -from .simd_kernels import ( - HAS_FAST_BYTE_SHUFFLE, - NIBBLE_TABLE_SIZE, - nibble_lookup, -) -from .simd_scan import clear_first_lane, first_lane_index, lane_bits +from .simd_kernels import HAS_FAST_BYTE_SHUFFLE from .static_bytes import table_bytes from .static_dfa import ( EDFA_EOL_AT_END, @@ -110,8 +105,6 @@ from .static_dfa import ( ) from .teddy import _lit_at -comptime _NibbleTable = SIMD[DType.uint8, NIBBLE_TABLE_SIZE] - # A 1-byte factor filters no better than the first-byte bitmap the # residual automaton already has, and drags every occurrence of a common # byte through a confirm walk. @@ -417,34 +410,6 @@ def _bucket_entries[ return out^ -def _rose_pos_masks[ - mn: Int, ln: Int -]( - meta: Array[Int32, mn], - lits: Array[Int32, ln], - n_entries: Int, - j: Int, -) -> Tuple[_NibbleTable, _NibbleTable]: - """Comptime: (lo, hi) nibble tables for factor byte position j; entry - bits are BUCKET indices. Caseless positions admit both cases (same low - nibble, both high nibbles). Mirrors _litset_pos_masks over the flat - pools.""" - var lo = _NibbleTable(0) - var hi = _NibbleTable(0) - for i in range(n_entries): - var base = _M_STRIDE * i - var packed = Int(lits[Int(meta[base + _M_BYTE_OFF]) + j]) - var b = packed & 0xFF - var bit = UInt8(1) << UInt8(Int(meta[base + _M_BUCKET])) - lo[b & 0x0F] |= bit - hi[b >> 4] |= bit - if (packed & 0x100) != 0: - var u = b - 32 # the uppercase member - lo[u & 0x0F] |= bit - hi[u >> 4] |= bit - return (lo, hi) - - # --- Factor extraction ------------------------------------------------------ @@ -1584,52 +1549,19 @@ def rose_scan[ lits: Array[Int32, ln], bcls: Array[Int32, bn], look: Array[Int32, kn], + masks: TeddyMasks, ](input: Span[Byte, origin]) -> List[SetMatch]: - """Scan for the factor-group patterns: Teddy front end, per-candidate - confirmation. Returns contract-ordered, deduped reports. + """Scan for the factor-group patterns: Teddy front end + (`teddy_front_end`, over `masks` = `litset_masks` of the factor set), + per-candidate confirmation. Returns contract-ordered, deduped reports. - Non-mutating; buffers are local. The chunk loop mirrors - litset_scan — lane shifts zero-fill, so the last k-1 lanes of each - chunk are re-examined as the head of the next. + Non-mutating; buffers are local. """ - comptime W = simd_width_of[DType.uint8]() - comptime k = min(3, r.min_len) - comptime m0 = _rose_pos_masks(meta, lits, r.n_entries, 0) - comptime m1 = _rose_pos_masks(meta, lits, r.n_entries, 1 if k > 1 else 0) - comptime m2 = _rose_pos_masks(meta, lits, r.n_entries, 2 if k > 2 else 0) - var out = List[SetMatch]() - var input_len = len(input) - var pos = 0 - var ptr = Pointer(input.unsafe_ptr()) - - while pos + W <= input_len: - var v = ptr.unsafe_offset(pos).unsafe_load[width=W]() - var lo = v & 0x0F - var hi = v >> 4 - var cand = nibble_lookup(m0[0], lo) & nibble_lookup(m0[1], hi) - comptime if k > 1: - var c1 = nibble_lookup(m1[0], lo) & nibble_lookup(m1[1], hi) - cand &= c1.shift_left[1]() - comptime if k > 2: - var c2 = nibble_lookup(m2[0], lo) & nibble_lookup(m2[1], hi) - cand &= c2.shift_left[2]() - var bits = lane_bits(cand.ne(0)) - while bits != 0: - var lane = first_lane_index(bits) - _rose_verify_at[ - r=r, - table=table, - flags=flags, - meta=meta, - lits=lits, - bcls=bcls, - look=look, - ](input, pos + lane, cand[lane], out) - bits = clear_first_lane(bits) - pos += W - (k - 1) - - while pos + r.min_len <= input_len: + var inp = Span[Byte, ImmOrigin(origin)](input) + + @always_inline + def verify(at: Int, bucket_mask: UInt8) {mut out, imm inp}: _rose_verify_at[ r=r, table=table, @@ -1638,9 +1570,9 @@ def rose_scan[ lits=lits, bcls=bcls, look=look, - ](input, pos, UInt8(0xFF), out) - pos += 1 + ](inp, at, bucket_mask, out) + teddy_front_end[min_len=r.min_len, masks=masks](inp, verify) sort_reports(out) dedup_reports(out) return out^ From 52cfc5db474200763f42377d0f2f29464a218803 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 15:09:19 -0600 Subject: [PATCH 51/65] cleanup: one bitnfa step shared by block and stream scans bitnfa_scan and bitnfa_stream_chunk repeated the per-byte gather / consume / exception table / shift / seed body; both now call the @always_inline _bn_consume and _bn_advance helpers (block mode still does not call the out-of-line stream function). Lane-probe asm and constant pools are identical to the previous build. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/set_bitnfa.mojo | 172 +++++++++++++++++-------------------- 1 file changed, 81 insertions(+), 91 deletions(-) diff --git a/emberregex/set_bitnfa.mojo b/emberregex/set_bitnfa.mojo index 9d4db3e..c50d67c 100644 --- a/emberregex/set_bitnfa.mojo +++ b/emberregex/set_bitnfa.mojo @@ -437,9 +437,6 @@ def build_bitnfa(nfa: NFA, enabled: Bool) -> BitNFA: return result^ -# --- Comptime materialization helpers --------------------------------------- - - # --- Runtime walker ---------------------------------------------------------- @@ -461,6 +458,80 @@ def _shl1[K: Int](v: SIMD[DType.uint64, K]) -> SIMD[DType.uint64, K]: return (v << 1) | (v >> 63).shift_right[1]() +@always_inline +def _bn_consume[ + rn: Int, //, d: BitNFA +]( + rch: Array[UInt64, rn], + b: Byte, + active: SIMD[DType.uint64, d.lanes], + gated: SIMD[DType.uint64, d.lanes], +) -> SIMD[DType.uint64, d.lanes]: + """The positions that consume `b`: the active set gathered through + the reach table, plus the newline-gated set when `b` is '\\n'.""" + comptime K = d.lanes + var rb = SIMD[DType.uint64, K](0) + comptime for l in range(K): + rb[l] = rch.unsafe_get(Int(b) * K + l) + + var consumed = active & rb + comptime if d.has_gated: + if b == CHAR_NEWLINE: + consumed |= gated & rb + return consumed + + +@always_inline +def _bn_advance[ + xn: Int, pn: Int, //, d: BitNFA +]( + exd: Array[UInt64, xn], + exi: Array[Int16, pn], + b: Byte, + consumed: SIMD[DType.uint64, d.lanes], + mut active: SIMD[DType.uint64, d.lanes], + mut gated: SIMD[DType.uint64, d.lanes], +): + """Advance past `b`: shift the limited chains, table the exceptions, + fold the restart seeds.""" + comptime K = d.lanes + comptime BitVec = SIMD[DType.uint64, K] + comptime limited_v = _bitvec[K](d.limited) + comptime exceptions_v = _bitvec[K](d.exceptions) + comptime seed_o_v = _bitvec[K](d.seed_other) + comptime seed_n_v = _bitvec[K](d.seed_nl) + comptime seed_go_v = _bitvec[K](d.seed_gated_other) + comptime seed_gn_v = _bitvec[K](d.seed_gated_nl) + + var nxt = _shl1(consumed & limited_v) + var gated_next = BitVec(0) + var ex = consumed & exceptions_v + if ex.reduce_or() != 0: + var is_nl = b == CHAR_NEWLINE + comptime for l in range(K): + var bits = ex[l] + while bits != 0: + var p = 64 * l + Int(count_trailing_zeros(bits)) + bits &= bits - 1 + var xi = Int(exi.unsafe_get(p)) + var base = xi * 4 * K + (K if is_nl else 0) + comptime for j in range(K): + nxt[j] |= exd.unsafe_get(base + j) + comptime if d.has_gated: + var gbase = xi * 4 * K + 2 * K + (K if is_nl else 0) + comptime for j in range(K): + gated_next[j] |= exd.unsafe_get(gbase + j) + + if b == CHAR_NEWLINE: + active = nxt | seed_n_v + comptime if d.has_gated: + gated = gated_next | seed_gn_v + else: + active = nxt | seed_o_v + comptime if d.has_gated: + gated = gated_next | seed_go_v + + @always_inline def _emit_bits[ pln: Int, @@ -552,16 +623,9 @@ def bitnfa_scan[ contract. Non-mutating; O(n * lanes) with exception work only when exception bits actually fire.""" comptime K = d.lanes - comptime BitVec = SIMD[DType.uint64, K] - comptime limited_v = _bitvec[K](d.limited) - comptime exceptions_v = _bitvec[K](d.exceptions) comptime accept_v = _bitvec[K](d.accept_union) comptime entry_v = _bitvec[K](d.entry) comptime entry_gated_v = _bitvec[K](d.entry_gated) - comptime seed_o_v = _bitvec[K](d.seed_other) - comptime seed_n_v = _bitvec[K](d.seed_nl) - comptime seed_go_v = _bitvec[K](d.seed_gated_other) - comptime seed_gn_v = _bitvec[K](d.seed_gated_nl) # Comptime arrays bound to the binary's constant data (no copy). var rch = materialize[reach]() @@ -577,14 +641,7 @@ def bitnfa_scan[ var pos = 0 while pos < input_len: var b = input.unsafe_get(pos) - var rb = BitVec(0) - comptime for l in range(K): - rb[l] = rch.unsafe_get(Int(b) * K + l) - - var consumed = active & rb - comptime if d.has_gated: - if b == CHAR_NEWLINE: - consumed |= gated & rb + var consumed = _bn_consume[d=d](rch, b, active, gated) # Reports: matches ending at pos + 1. var acc = consumed & accept_v @@ -596,35 +653,7 @@ def bitnfa_scan[ acc, b, pos + 1, nb, ids, out ) - # Advance: shift the limited chains, table the exceptions, - # fold the restart seeds. - var nxt = _shl1(consumed & limited_v) - var gated_next = BitVec(0) - var ex = consumed & exceptions_v - if ex.reduce_or() != 0: - var is_nl = b == CHAR_NEWLINE - comptime for l in range(K): - var bits = ex[l] - while bits != 0: - var p = 64 * l + Int(count_trailing_zeros(bits)) - bits &= bits - 1 - var xi = Int(exi.unsafe_get(p)) - var base = xi * 4 * K + (K if is_nl else 0) - comptime for j in range(K): - nxt[j] |= exd.unsafe_get(base + j) - comptime if d.has_gated: - var gbase = xi * 4 * K + 2 * K + (K if is_nl else 0) - comptime for j in range(K): - gated_next[j] |= exd.unsafe_get(gbase + j) - - if b == CHAR_NEWLINE: - active = nxt | seed_n_v - comptime if d.has_gated: - gated = gated_next | seed_gn_v - else: - active = nxt | seed_o_v - comptime if d.has_gated: - gated = gated_next | seed_go_v + _bn_advance[d=d](exd, exi, b, consumed, active, gated) pos += 1 return out^ @@ -719,18 +748,12 @@ def bitnfa_stream_chunk[ ): """Consume one chunk, appending reports at GLOBAL offsets. - Mirrors `bitnfa_scan`'s loop exactly; the only differences are the - global offset and the one-step report delay. + `bitnfa_scan`'s loop (the same `_bn_consume` / `_bn_advance` step); + the only differences are the global offset and the one-step report + delay. """ comptime K = d.lanes - comptime BitVec = SIMD[DType.uint64, K] - comptime limited_v = _bitvec[K](d.limited) - comptime exceptions_v = _bitvec[K](d.exceptions) comptime accept_v = _bitvec[K](d.accept_union) - comptime seed_o_v = _bitvec[K](d.seed_other) - comptime seed_n_v = _bitvec[K](d.seed_nl) - comptime seed_go_v = _bitvec[K](d.seed_gated_other) - comptime seed_gn_v = _bitvec[K](d.seed_gated_nl) # Only EOL-sensitive sets need the one-step delay: `nl` and `end` # slices resolve against the byte AFTER the match. Everything else @@ -751,14 +774,7 @@ def bitnfa_stream_chunk[ st, Int(b), ids, out ) - var rb = BitVec(0) - comptime for l in range(K): - rb[l] = rch.unsafe_get(Int(b) * K + l) - - var consumed = st.active & rb - comptime if d.has_gated: - if b == CHAR_NEWLINE: - consumed |= st.gated & rb + var consumed = _bn_consume[d=d](rch, b, st.active, st.gated) var acc = consumed & accept_v comptime if NEEDS_DELAY: @@ -772,33 +788,7 @@ def bitnfa_stream_chunk[ acc, b, st.offset + pos + 1, -1, ids, out ) - var nxt = _shl1(consumed & limited_v) - var gated_next = BitVec(0) - var ex = consumed & exceptions_v - if ex.reduce_or() != 0: - var is_nl = b == CHAR_NEWLINE - comptime for l in range(K): - var bits = ex[l] - while bits != 0: - var p = 64 * l + Int(count_trailing_zeros(bits)) - bits &= bits - 1 - var xi = Int(exi.unsafe_get(p)) - var base = xi * 4 * K + (K if is_nl else 0) - comptime for j in range(K): - nxt[j] |= exd.unsafe_get(base + j) - comptime if d.has_gated: - var gbase = xi * 4 * K + 2 * K + (K if is_nl else 0) - comptime for j in range(K): - gated_next[j] |= exd.unsafe_get(gbase + j) - - if b == CHAR_NEWLINE: - st.active = nxt | seed_n_v - comptime if d.has_gated: - st.gated = gated_next | seed_gn_v - else: - st.active = nxt | seed_o_v - comptime if d.has_gated: - st.gated = gated_next | seed_go_v + _bn_advance[d=d](exd, exi, b, consumed, st.active, st.gated) pos += 1 st.offset += input_len From 171e497ba2528cf5719ce9b6aed26ec8c8323c5f Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 15:11:12 -0600 Subject: [PATCH 52/65] cleanup: drop the unreachable shared-entry reverse start scan build_union_subset_nfa and splice_nfa give every pattern a fresh entry state (each fragment start is a state its own build appended), so the dup/-2 detection in _start_id_map and the exact _start_ids scan it selected could only be reached by hand-editing pattern_starts, which is all test_reverse_dfa_shared_entry_uses_exact_start_scan did. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/set_reverse.mojo | 55 +++++++++++-------------------------- test/test_set_paths.mojo | 16 ----------- 2 files changed, 16 insertions(+), 55 deletions(-) diff --git a/emberregex/set_reverse.mojo b/emberregex/set_reverse.mojo index 9450bd9..1e49ea2 100644 --- a/emberregex/set_reverse.mojo +++ b/emberregex/set_reverse.mojo @@ -192,41 +192,22 @@ def _rev_closure( sort(out) -def _start_ids(nfa: NFA, states: List[Int]) -> List[Int]: - """Comptime: ids whose fragment entry is live in this set.""" - var ids = List[Int]() - for i in range(len(nfa.pattern_starts)): - var st = nfa.pattern_starts[i] - if st < 0: - continue - for s in states: - if s == st: - ids.append(i) - break - return _sorted_dedup(ids^) - - def _start_id_map(nfa: NFA) -> List[Int]: """Comptime: NFA state -> the pattern id whose fragment entry it is, - -1 elsewhere; -2 in slot 0 when two patterns share an entry state - (then `_start_ids` must do the exact scan).""" + -1 elsewhere. One id per state suffices: the union builders + (`build_union_subset_nfa`, `splice_nfa`) give every pattern a fresh + entry state.""" var m = List[Int](fill=-1, length=len(nfa.states)) - var dup = False for i in range(len(nfa.pattern_starts)): var st = nfa.pattern_starts[i] - if st < 0: - continue - if m[st] >= 0: - dup = True - m[st] = i - if dup and len(m) > 0: - m[0] = -2 + if st >= 0: + m[st] = i return m^ def _start_ids_mapped(start_id_of: List[Int], states: List[Int]) -> List[Int]: - """`_start_ids` in O(members): one map read per member. Same ids in - the same (sorted, deduplicated) order.""" + """Comptime: ids whose fragment entry is live in this set, sorted and + deduplicated — one map read per member.""" var ids = List[Int]() for s in states: var i = start_id_of[s] @@ -750,22 +731,18 @@ def _rdfa_finish( result.bol0_len = List[Int](fill=0, length=n) result.bolnl_off = List[Int](fill=0, length=n) result.bolnl_len = List[Int](fill=0, length=n) - # One NFA-state -> pattern-id map for the whole finish: `_start_ids` - # scanned every pattern against every member per state, O(patterns x - # members) List reads — ~17 s for a 456-state reverse DFA over 100 - # literals. The BOL walks are skipped outright when the union has no - # BOL anchor (they return empty then): they allocate a visited array - # per member anchor and walk the predecessor lists. + # One NFA-state -> pattern-id map for the whole finish: scanning every + # pattern against every member per state was O(patterns x members) + # List reads — ~17 s for a 456-state reverse DFA over 100 literals. + # The BOL walks are skipped outright when the union has no BOL anchor + # (they return empty then): they allocate a visited array per member + # anchor and walk the predecessor lists. var start_id_of = _start_id_map(nfa) - var exact_starts = len(start_id_of) > 0 and start_id_of[0] == -2 var has_bol = _nfa_has_bol(nfa) for s in range(n): - var ids: List[Int] - if exact_starts: - ids = _start_ids(nfa, sets[s]) - else: - ids = _start_ids_mapped(start_id_of, sets[s]) - var sn = _pool_slice(result.pool, ids) + var sn = _pool_slice( + result.pool, _start_ids_mapped(start_id_of, sets[s]) + ) result.norm_off[s] = sn[0] result.norm_len[s] = sn[1] var s0 = _pool_slice( diff --git a/test/test_set_paths.mojo b/test/test_set_paths.mojo index e4372c0..eba56c6 100644 --- a/test/test_set_paths.mojo +++ b/test/test_set_paths.mojo @@ -438,22 +438,6 @@ def test_reverse_dfa_bol_slices() raises: assert_equal(len(never.pool), 1) -def test_reverse_dfa_shared_entry_uses_exact_start_scan() raises: - # Two ids whose fragments share an entry state defeat the one-id-per- - # state map, so the finish falls back to the exact per-pattern scan - # and the shared state's norm slice carries BOTH ids. - var nfa = build_union_nfa(["ab", "cd"]) - nfa.pattern_starts[1] = nfa.pattern_starts[0] - var rd = build_reverse_dfa(nfa, True) - assert_true(rd.valid) - var shared = 0 - for s in range(rd.num_states): - if rd.norm_len[s] == 2: - shared += 1 - assert_equal(_ids(rd.pool, rd.norm_off[s], 2), [0, 1]) - assert_equal(shared, 1) - - def test_reverse_dfa_caps() raises: # Reversed, `[ab]{10}a[ab]*` is `[ab]*a[ab]{10}` anchored at its # start: the forward table is small, the reverse one blows From 199fccee55a9482c4190da214e0c0b314f6d189c Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 15:13:18 -0600 Subject: [PATCH 53/65] cleanup: drop dead template params from _rose_walk / _rose_confirm _rose_walk never read meta, lits or bcls, and _rose_confirm took lits and bcls only to pass them on. Lane-probe asm unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/set_rose.mojo | 32 ++------------------------------ 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/emberregex/set_rose.mojo b/emberregex/set_rose.mojo index 1c736e1..9b1246a 100644 --- a/emberregex/set_rose.mojo +++ b/emberregex/set_rose.mojo @@ -1322,16 +1322,10 @@ def _look_ok[ def _rose_walk[ origin: Origin, fn_: Int, - mn: Int, - ln: Int, - bn: Int, //, r: RoseView, table: StringLiteral, flags: Array[UInt8, fn_], - meta: Array[Int32, mn], - lits: Array[Int32, ln], - bcls: Array[Int32, bn], pid: Int, ]( input: Span[Byte, origin], @@ -1380,15 +1374,11 @@ def _rose_confirm[ origin: Origin, fn_: Int, mn: Int, - ln: Int, - bn: Int, //, r: RoseView, table: StringLiteral, flags: Array[UInt8, fn_], meta: Array[Int32, mn], - lits: Array[Int32, ln], - bcls: Array[Int32, bn], entry: Int, pid: Int, ](input: Span[Byte, origin], start: Int, mut out: List[SetMatch]): @@ -1404,15 +1394,7 @@ def _rose_confirm[ cur = s_nl else: cur = s_other - _rose_walk[ - r=r, - table=table, - flags=flags, - meta=meta, - lits=lits, - bcls=bcls, - pid=pid, - ](input, start, cur, out) + _rose_walk[r=r, table=table, flags=flags, pid=pid](input, start, cur, out) @always_inline @@ -1476,8 +1458,6 @@ def _rose_verify_at[ table=table, flags=flags, meta=meta, - lits=lits, - bcls=bcls, entry=i, pid=pid, ](input, s0, out) @@ -1500,13 +1480,7 @@ def _rose_verify_at[ cur = k2 if cur >= 0: _rose_walk[ - r=r, - table=table, - flags=flags, - meta=meta, - lits=lits, - bcls=bcls, - pid=pid, + r=r, table=table, flags=flags, pid=pid ](input, at + L, cur, out) elif not skip: if ( @@ -1521,8 +1495,6 @@ def _rose_verify_at[ table=table, flags=flags, meta=meta, - lits=lits, - bcls=bcls, entry=i, pid=pid, ](input, at - off, out) From 30f656537389e7e97ef4f5a8bb0b5b19321a18ed Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 15:16:19 -0600 Subject: [PATCH 54/65] cleanup: one report sort+dedup pair next to SetMatch set_literal's _sort_reports + inline dedup and set_rose's sort_reports + dedup_reports were the same insertion sort (identical predicate) and the same compaction; both lanes now call the Rose pair, moved to set_pike.mojo. The moved bodies are asm-identical; the Teddy lane now calls them out of line instead of inlining its private copy. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/set_literal.mojo | 31 +++++------------------------ emberregex/set_pike.mojo | 37 +++++++++++++++++++++++++++++++++++ emberregex/set_rose.mojo | 39 +------------------------------------ 3 files changed, 43 insertions(+), 64 deletions(-) diff --git a/emberregex/set_literal.mojo b/emberregex/set_literal.mojo index 313c655..d8774ca 100644 --- a/emberregex/set_literal.mojo +++ b/emberregex/set_literal.mojo @@ -25,7 +25,7 @@ from std.sys import simd_width_of from .nfa import NFA, NFAStateKind from .optimize import _charset_filter_byte -from .set_pike import SetMatch +from .set_pike import SetMatch, dedup_reports, sort_reports from .simd_kernels import NIBBLE_TABLE_SIZE, nibble_lookup from .simd_scan import clear_first_lane, first_lane_index, lane_bits from .teddy import _lit_at @@ -396,21 +396,6 @@ def _litset_verify_at[ out.append(SetMatch(rid, at + L)) -def _sort_reports(mut r: List[SetMatch]): - """Order reports by (end, id). The scan emits grouped by - nondecreasing start, so displacement is bounded by the literal - length spread and insertion sort runs near-linear.""" - for i in range(1, len(r)): - var key = r[i] - var j = i - 1 - while j >= 0 and ( - r[j].end > key.end or (r[j].end == key.end and r[j].id > key.id) - ): - r[j + 1] = r[j] - j -= 1 - r[j + 1] = key - - # `@always_inline` for the same reason as `mdfa_scan` and the eager # walkers: `ls` is a List-carrying value parameter, and an out-of-line # instantiation prints it into its symbol name (a 1 MB symbol for the @@ -431,14 +416,8 @@ def litset_scan[ teddy_front_end[min_len=ls.min_len, masks=litset_masks(ls)](inp, verify) - _sort_reports(out) - # Collapse duplicate (id, end) pairs: same-id arms of an in-pattern - # alternation (`ab|ab`-style) can hit at the same end. - var w = 0 - for i in range(len(out)): - if w > 0 and out[i] == out[w - 1]: - continue - out[w] = out[i] - w += 1 - out.resize(w, SetMatch(0, 0)) + sort_reports(out) + # Same-id arms of an in-pattern alternation (`ab|ab`-style) can hit at + # the same end. + dedup_reports(out) return out^ diff --git a/emberregex/set_pike.mojo b/emberregex/set_pike.mojo index 54fa08a..46d05d6 100644 --- a/emberregex/set_pike.mojo +++ b/emberregex/set_pike.mojo @@ -67,6 +67,43 @@ struct SetSpan(Equatable, TrivialRegisterPassable, Writable): ) +@always_inline +def _before(a: SetMatch, b: SetMatch) -> Bool: + """Contract order: nondecreasing end, ties ascending id.""" + return a.end < b.end or (a.end == b.end and a.id <= b.id) + + +def sort_reports(mut r: List[SetMatch]): + """Order reports by (end, id). + + The Teddy and Rose scans emit reports grouped by candidate start, and + candidates advance monotonically, so displacement is bounded — by the + literal length spread, or by how far one Rose confirm walk can reach + past the next candidate (`_conf_dfa_ok` keeps the far-walking ones off + that lane). Insertion sort is near-linear there and allocates nothing, + which matters: on dense input this runs over thousands of reports per + scan. + """ + for i in range(1, len(r)): + var key = r[i] + var j = i - 1 + while j >= 0 and not _before(r[j], key): + r[j + 1] = r[j] + j -= 1 + r[j + 1] = key + + +def dedup_reports(mut r: List[SetMatch]): + """Collapse duplicate (id, end) pairs in a sorted report list.""" + var w = 0 + for i in range(len(r)): + if w > 0 and r[i] == r[w - 1]: + continue + r[w] = r[i] + w += 1 + r.resize(w, SetMatch(0, 0)) + + def _set_add_state[ origin: Origin, // ]( diff --git a/emberregex/set_rose.mojo b/emberregex/set_rose.mojo index 9b1246a..e596682 100644 --- a/emberregex/set_rose.mojo +++ b/emberregex/set_rose.mojo @@ -86,7 +86,7 @@ from .set_literal import ( _assign_buckets, teddy_front_end, ) -from .set_pike import SetMatch +from .set_pike import SetMatch, _before, dedup_reports, sort_reports from .set_semantics import ( EXT_EDIT_DISTANCE, EXT_HAMMING_DISTANCE, @@ -1193,43 +1193,6 @@ def rose_flags_arr[n: Int](r: RoseSet) -> Array[UInt8, n]: # --- Report ordering -------------------------------------------------------- -@always_inline -def _before(a: SetMatch, b: SetMatch) -> Bool: - """Contract order: nondecreasing end, ties ascending id.""" - return a.end < b.end or (a.end == b.end and a.id <= b.id) - - -def sort_reports(mut r: List[SetMatch]): - """Order reports by (end, id). - - Reports leave the scan grouped by candidate start, and candidates - advance monotonically, so displacement is bounded by how far one - confirm walk can reach past the next candidate — a few bytes for the - literal-ish patterns this lane accepts (`_conf_dfa_ok` already keeps - the far-walking ones off it). Insertion sort is near-linear there and - allocates nothing, which matters: on dense input this runs over - thousands of reports per scan. - """ - for i in range(1, len(r)): - var key = r[i] - var j = i - 1 - while j >= 0 and not _before(r[j], key): - r[j + 1] = r[j] - j -= 1 - r[j + 1] = key - - -def dedup_reports(mut r: List[SetMatch]): - """Collapse duplicate (id, end) pairs in a sorted report list.""" - var w = 0 - for i in range(len(r)): - if w > 0 and r[i] == r[w - 1]: - continue - r[w] = r[i] - w += 1 - r.resize(w, SetMatch(0, 0)) - - def merge_reports(var a: List[SetMatch], b: List[SetMatch]) -> List[SetMatch]: """Merge two contract-ordered report lists, collapsing duplicates.""" var out = List[SetMatch](capacity=len(a) + len(b)) From 8f5e45b9bd3762086c0f2a5882fa0cf0ce4f7af4 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 15:18:33 -0600 Subject: [PATCH 55/65] cleanup: set_semantics dead param, any_ext reuse, dead guard needs_som never read its flags param; has_semantics re-inlined any_ext; sem_table_arr's `b + SEM_MIN_LEN >= n` break could not fire (its only caller sizes n = sem_table_len = SEM_STRIDE * num_patterns). apply_semantics / apply_semantics_spans stay separate: SetMatch has no start field, so one generic needs a trait or per-branch rebinds, which is no shorter than the two 10-line loops. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/set_engine.mojo | 2 +- emberregex/set_semantics.mojo | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/emberregex/set_engine.mojo b/emberregex/set_engine.mojo index 5141f37..d349619 100644 --- a/emberregex/set_engine.mojo +++ b/emberregex/set_engine.mojo @@ -453,7 +453,7 @@ struct RegexSet[ # --- Semantic surface (phase 7) ---------------------------------------- comptime _has_sem = has_semantics(Self.flags, Self.ext, Self.num_patterns) - comptime _sem_needs_som = needs_som(Self.flags, Self.ext, Self.num_patterns) + comptime _sem_needs_som = needs_som(Self.ext, Self.num_patterns) comptime _SEM = sem_table_arr[sem_table_len(Self.num_patterns)]( Self.flags, Self.ext, Self.num_patterns ) diff --git a/emberregex/set_semantics.mojo b/emberregex/set_semantics.mojo index c7768b6..e099278 100644 --- a/emberregex/set_semantics.mojo +++ b/emberregex/set_semantics.mojo @@ -87,7 +87,7 @@ def any_ext(ext: List[Int], field: Int, num_patterns: Int) -> Bool: return False -def needs_som(flags: List[Int], ext: List[Int], num_patterns: Int) -> Bool: +def needs_som(ext: List[Int], num_patterns: Int) -> Bool: """Comptime: does the semantic surface force start-of-match? Only `min_length` does — it constrains the match WIDTH, which the @@ -101,12 +101,11 @@ def has_semantics(flags: List[Int], ext: List[Int], num_patterns: Int) -> Bool: pass is compiled out.""" if any_flag(flags, SetFlags.SINGLEMATCH | SetFlags.QUIET): return True - for i in range(num_patterns): - # Only the first three are report-stream filters; the distances - # are applied to the automaton itself at build time. - for f in range(3): - if ext_of(ext, i, f) >= 0: - return True + # Only the first three are report-stream filters; the distances are + # applied to the automaton itself at build time. + for f in range(3): + if any_ext(ext, f, num_patterns): + return True return False @@ -262,8 +261,6 @@ def sem_table_arr[ var arr = Array[Int32, n](fill=-1) for i in range(num_patterns): var b = SEM_STRIDE * i - if b + SEM_MIN_LEN >= n: - break arr[b + SEM_FLAG] = Int32(flag_of(flags, i)) arr[b + SEM_MIN_OFF] = Int32(ext_of(ext, i, EXT_MIN_OFFSET)) arr[b + SEM_MAX_OFF] = Int32(ext_of(ext, i, EXT_MAX_OFFSET)) From 60b57d37516f40243445c783975217b9d5d60792 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 15:26:42 -0600 Subject: [PATCH 56/65] cleanup: drop set-lane alias decls and one-off table converters - _SBN_* / _stream_bn / _can_stream only renamed _BN_* / _bitnfa / _bitnfa.valid; set_stream.mojo and test_set_phase6 read those. - rebind[NFA](self._nfa) was a no-op now that _nfa is a plain NFA. - mdfa_table_str / rdfa_table_str / rose_table_str were pure table_bytes wrappers, rose_flags_arr / ac_cls_arr pure int_arr copies; the decl-level sites (and the bench/tests that used mdfa_table_str) call table_bytes / int_arr directly. Lane-probe asm and static data are identical to the previous build. Co-Authored-By: Claude Opus 5.5 (1M context) --- bench/bench_set.mojo | 7 ++--- emberregex/set_ac.mojo | 8 ------ emberregex/set_dfa.mojo | 11 ++------ emberregex/set_engine.mojo | 45 +++++++++++-------------------- emberregex/set_reverse.mojo | 7 ----- emberregex/set_rose.mojo | 21 ++------------- emberregex/set_stream.mojo | 28 +++++++++---------- test/test_set_ac.mojo | 3 +-- test/test_set_bench_coverage.mojo | 7 ++--- test/test_set_phase2.mojo | 7 ++--- test/test_set_phase6.mojo | 4 +-- 11 files changed, 48 insertions(+), 100 deletions(-) diff --git a/bench/bench_set.mojo b/bench/bench_set.mojo index b9a68f5..b894eb3 100644 --- a/bench/bench_set.mojo +++ b/bench/bench_set.mojo @@ -40,9 +40,8 @@ from emberregex.set_dfa import ( build_multi_dfa, mdfa_scan, mdfa_slices_arr, - mdfa_table_str, ) -from emberregex.static_bytes import int_arr, list_arr, static_bytes +from emberregex.static_bytes import int_arr, list_arr, static_bytes, table_bytes # --------------------------------------------------------------------------- @@ -191,7 +190,9 @@ def mdfa_direct_scan[ engine selection (which now sends these sets to Rose).""" comptime S = RegexSet[patterns] comptime MD = build_multi_dfa(S.nfa, S.nfa.can_use_dfa) - comptime T = static_bytes[mdfa_table_str[MD.num_states * 256](MD)]() + comptime T = static_bytes[ + table_bytes[DType.int16](MD.table, MD.num_states * 256) + ]() comptime P = int_arr[DType.int32, len(MD.pool)](MD.pool, 0) comptime SL = mdfa_slices_arr[6 * MD.num_states](MD) return mdfa_scan[d=MD, table=T, pool=P, slices=SL](input) diff --git a/emberregex/set_ac.mojo b/emberregex/set_ac.mojo index 1c8e0e3..c3a6a8b 100644 --- a/emberregex/set_ac.mojo +++ b/emberregex/set_ac.mojo @@ -490,14 +490,6 @@ def ac_table_str[n: Int](d: ACSet) -> String: return out^ -def ac_cls_arr(d: ACSet) -> Array[UInt8, 256]: - """Byte -> class map. Class 0 collects every byte no literal uses.""" - var arr = Array[UInt8, 256](fill=0) - for b in range(256): - arr[b] = UInt8(d.class_map[b]) - return arr^ - - def ac_rep_arr[n: Int](d: ACSet) -> Array[Int32, n]: """Per-state report slice, interleaved as (offset, length) so the hot-path length test and the offset it needs share a cache line.""" diff --git a/emberregex/set_dfa.mojo b/emberregex/set_dfa.mojo index 6ec9474..5ede463 100644 --- a/emberregex/set_dfa.mojo +++ b/emberregex/set_dfa.mojo @@ -45,7 +45,6 @@ from .simd_kernels import ( nibble_table_from, ) from .simd_scan import first_lane_index, lane_bits -from .static_bytes import table_bytes from .static_dfa import ( EDFA_NFA_CAP, _FlatNFA, @@ -69,8 +68,8 @@ comptime MDFA_STATE_CAP = 512 struct MultiDFA(Copyable, Movable): """Comptime-computed multi-accept DFA. Only ever exists as a - comptime value; the runtime walker reads the materialized - Array forms (mdfa_*_arr).""" + comptime value; the runtime walker reads the materialized forms + (the `table_bytes` string, the pool, `mdfa_slices_arr`).""" var valid: Bool var num_states: Int @@ -643,12 +642,6 @@ def _build_multi_dfa_list(nfa: NFA) -> MultiDFA: # --- Comptime materialization helpers --------------------------------------- -def mdfa_table_str[n: Int](d: MultiDFA) -> String: - """Int16 state ids (MDFA_STATE_CAP < 32768 by construction) as `n` - little-endian entries; see static_bytes.mojo for why a string.""" - return table_bytes[DType.int16](d.table, n) - - def mdfa_slices_arr[n: Int](d: MultiDFA) -> Array[Int32, n]: """Per-state slice metadata, interleaved as 6 Int32 per state: (norm_off, norm_len, nl_off, nl_len, end_off, end_len).""" diff --git a/emberregex/set_engine.mojo b/emberregex/set_engine.mojo index d349619..44195c1 100644 --- a/emberregex/set_engine.mojo +++ b/emberregex/set_engine.mojo @@ -49,9 +49,8 @@ from std.math import max from std.os import abort from .nfa import NFA -from .static_bytes import int_arr, list_arr, static_bytes +from .static_bytes import int_arr, list_arr, static_bytes, table_bytes from .set_ac import ( - ac_cls_arr, ac_rep_arr, ac_scan, ac_table_str, @@ -66,7 +65,6 @@ from .set_dfa import ( build_multi_dfa, mdfa_scan, mdfa_slices_arr, - mdfa_table_str, ) from .set_literal import extract_literal_set, litset_masks, litset_scan from .set_nfa import build_union_nfa, build_union_subset_nfa, _union_any_unicode @@ -80,7 +78,6 @@ from .set_pike import ( from .set_rose import ( build_rose, merge_reports, - rose_flags_arr, rose_bcls_arr, _rose_lits, _rose_meta, @@ -90,14 +87,12 @@ from .set_rose import ( rose_look_len, rose_meta_len, rose_scan, - rose_table_str, rose_view, ) from .set_reverse import ( build_reverse_dfa, leftmost_nonoverlapping, rdfa_slices_arr, - rdfa_table_str, rdfa_view, reverse_som, ) @@ -250,7 +245,7 @@ struct RegexSet[ comptime _AC_TABLE_S = ac_table_str[ Self._ac.num_states * Self._ac.num_classes ](Self._ac) - comptime _AC_CLS = ac_cls_arr(Self._ac) + comptime _AC_CLS = int_arr[DType.uint8, 256](Self._ac.class_map, 0) comptime _AC_REP = ac_rep_arr[2 * Self._ac.num_states](Self._ac) comptime _AC_POOL = int_arr[DType.int32, len(Self._ac.pool)]( Self._ac.pool, 0 @@ -266,11 +261,11 @@ struct RegexSet[ ) comptime _use_rose = Self._rose.valid comptime _ROSE_TABLE = static_bytes[Self._ROSE_TABLE_S]() - comptime _ROSE_TABLE_S = rose_table_str[Self._rose.num_conf_states * 256]( - Self._rose + comptime _ROSE_TABLE_S = table_bytes[DType.int32]( + Self._rose.conf_table, Self._rose.num_conf_states * 256 ) - comptime _ROSE_FLAGS = rose_flags_arr[Self._rose.num_conf_states]( - Self._rose + comptime _ROSE_FLAGS = int_arr[DType.uint8, Self._rose.num_conf_states]( + Self._rose.conf_flags, 0 ) # The walkers take the table-free view: comptime parameter values are # mangled into symbol names, and carrying the confirm table there too @@ -301,8 +296,8 @@ struct RegexSet[ ) comptime _use_res_mdfa = Self._res_mdfa.valid comptime _RES_TABLE = static_bytes[Self._RES_TABLE_S]() - comptime _RES_TABLE_S = mdfa_table_str[Self._res_mdfa.num_states * 256]( - Self._res_mdfa + comptime _RES_TABLE_S = table_bytes[DType.int16]( + Self._res_mdfa.table, Self._res_mdfa.num_states * 256 ) comptime _RES_POOL = int_arr[DType.int32, len(Self._res_mdfa.pool)]( Self._res_mdfa.pool, 0 @@ -350,8 +345,9 @@ struct RegexSet[ ) comptime _use_mdfa = Self._mdfa.valid comptime _MDFA_TABLE = static_bytes[Self._MDFA_TABLE_S]() - comptime _MDFA_TABLE_S = mdfa_table_str[Self._mdfa.num_states * 256]( - Self._mdfa + # Int16 state ids: MDFA_STATE_CAP < 32768 by construction. + comptime _MDFA_TABLE_S = table_bytes[DType.int16]( + Self._mdfa.table, Self._mdfa.num_states * 256 ) comptime _MDFA_POOL = int_arr[DType.int32, len(Self._mdfa.pool)]( Self._mdfa.pool, 0 @@ -368,8 +364,6 @@ struct RegexSet[ # cliff, so this costs linear comptime work even when a faster lane # owns `scan`. comptime _bitnfa = build_bitnfa(Self.nfa, Self.nfa.can_use_dfa) - comptime _can_stream = Self._bitnfa.valid - comptime _stream_bn = Self._bitnfa # Cheap lane predicates FIRST: comptime `and` short-circuits during # elaboration, so testing `_bitnfa.valid` last means a set that Teddy, # Rose or the multi-DFA already owns never runs the LimEx construction at @@ -398,13 +392,6 @@ struct RegexSet[ comptime _BN_SLICES = int_arr[DType.int32, 12 * Self._bitnfa.num_positions]( Self._bitnfa.slices, 0 ) - # Stream aliases: same arrays, named for the streaming API so - # set_stream.mojo does not reach into block-lane internals. - comptime _SBN_REACH = Self._BN_REACH - comptime _SBN_EX = Self._BN_EX - comptime _SBN_EXIDX = Self._BN_EXIDX - comptime _SBN_POOL = Self._BN_POOL - comptime _SBN_SLICES = Self._BN_SLICES comptime _use_pike = ( not Self._use_litset @@ -430,8 +417,8 @@ struct RegexSet[ comptime _use_rdfa = Self._rdfa.valid comptime _rdfa_v = rdfa_view(Self._rdfa) comptime _RD_TABLE = static_bytes[Self._RD_TABLE_S]() - comptime _RD_TABLE_S = rdfa_table_str[Self._rdfa.num_states * 256]( - Self._rdfa + comptime _RD_TABLE_S = table_bytes[DType.int32]( + Self._rdfa.table, Self._rdfa.num_states * 256 ) comptime _RD_POOL = int_arr[DType.int32, len(Self._rdfa.pool)]( Self._rdfa.pool, 0 @@ -562,8 +549,7 @@ struct RegexSet[ i += 1 return self._confirm(input, out^) else: - ref nfa = rebind[NFA](self._nfa) - return self._confirm(input, set_pike_som_scan(nfa, input)) + return self._confirm(input, set_pike_som_scan(self._nfa, input)) def _scan_residual[ origin: Origin, // @@ -773,5 +759,4 @@ struct RegexSet[ slices=Self._BN_SLICES, ](input) else: - ref nfa = rebind[NFA](self._nfa) - return set_pike_scan(nfa, input) + return set_pike_scan(self._nfa, input) diff --git a/emberregex/set_reverse.mojo b/emberregex/set_reverse.mojo index 1e49ea2..9cc5d93 100644 --- a/emberregex/set_reverse.mojo +++ b/emberregex/set_reverse.mojo @@ -56,7 +56,6 @@ from .set_dfa import ( new_set_index, ) from .set_pike import SetSpan -from .static_bytes import table_bytes from .static_dfa import ( EDFA_NFA_CAP, _FlatNFA, @@ -776,12 +775,6 @@ def _rdfa_finish( # --- Comptime materialization helpers --------------------------------------- -def rdfa_table_str[n: Int](d: ReverseDFA) -> String: - """The flat table as `n` little-endian Int32 entries; see - static_bytes.mojo for why a string.""" - return table_bytes[DType.int32](d.table, n) - - def rdfa_slices_arr[n: Int](d: ReverseDFA) -> Array[Int32, n]: """Per-state slice metadata, 6 Int32 per state: (norm_off, norm_len, bol0_off, bol0_len, bolnl_off, bolnl_len).""" diff --git a/emberregex/set_rose.mojo b/emberregex/set_rose.mojo index e596682..d8988d8 100644 --- a/emberregex/set_rose.mojo +++ b/emberregex/set_rose.mojo @@ -93,7 +93,6 @@ from .set_semantics import ( ext_of, ) from .simd_kernels import HAS_FAST_BYTE_SHUFFLE -from .static_bytes import table_bytes from .static_dfa import ( EDFA_EOL_AT_END, EDFA_EOL_AT_NEWLINE, @@ -151,8 +150,8 @@ comptime ROSE_CONF_STATE_CAP = 512 struct RoseSet(Copyable, Movable): """Comptime-computed decomposition. Only ever exists as a comptime - value; the runtime walker reads the materialized Array forms - (rose_table_arr / rose_flags_arr). + value; the runtime walker reads the materialized forms (RegexSet's + `_ROSE_*` decls). `lit` is entry-indexed and holds exactly what the phase-1 Teddy machinery needs (bytes, caseless flags, report ids, buckets); @@ -1174,22 +1173,6 @@ def build_rose( return result^ -# --- Comptime materialization helpers --------------------------------------- - - -def rose_table_str[n: Int](r: RoseSet) -> String: - """The concatenated confirm tables as `n` little-endian Int32 - entries; see static_bytes.mojo for why a string.""" - return table_bytes[DType.int32](r.conf_table, n) - - -def rose_flags_arr[n: Int](r: RoseSet) -> Array[UInt8, n]: - var arr = Array[UInt8, n](fill=0) - for i in range(n): - arr[i] = UInt8(r.conf_flags[i]) - return arr^ - - # --- Report ordering -------------------------------------------------------- diff --git a/emberregex/set_stream.mojo b/emberregex/set_stream.mojo index ddc0f31..0d57338 100644 --- a/emberregex/set_stream.mojo +++ b/emberregex/set_stream.mojo @@ -128,12 +128,12 @@ struct SetStream[patterns: List[String], allow_empty: Bool = False]( """ comptime _db = RegexSet[Self.patterns, Self.allow_empty] - comptime _bn = Self._db._stream_bn + comptime _bn = Self._db._bitnfa comptime _K = Self._bn.lanes comptime _stream_ok = _check_streamable( Self.patterns, Self._db.nfa.can_use_dfa, - Self._db._can_stream, + Self._db._bitnfa.valid, Self._db._needs_confirm, Self._db._confirm_ids, ) @@ -164,11 +164,11 @@ struct SetStream[patterns: List[String], allow_empty: Bool = False]( var out = List[SetMatch]() bitnfa_stream_chunk[ d=Self._bn, - reach=Self._db._SBN_REACH, - ex_data=Self._db._SBN_EX, - ex_idx=Self._db._SBN_EXIDX, - pool=Self._db._SBN_POOL, - slices=Self._db._SBN_SLICES, + reach=Self._db._BN_REACH, + ex_data=Self._db._BN_EX, + ex_idx=Self._db._BN_EXIDX, + pool=Self._db._BN_POOL, + slices=Self._db._BN_SLICES, ](self._st, input, out) return out^ @@ -181,11 +181,11 @@ struct SetStream[patterns: List[String], allow_empty: Bool = False]( for c in chunks: bitnfa_stream_chunk[ d=Self._bn, - reach=Self._db._SBN_REACH, - ex_data=Self._db._SBN_EX, - ex_idx=Self._db._SBN_EXIDX, - pool=Self._db._SBN_POOL, - slices=Self._db._SBN_SLICES, + reach=Self._db._BN_REACH, + ex_data=Self._db._BN_EX, + ex_idx=Self._db._BN_EXIDX, + pool=Self._db._BN_POOL, + slices=Self._db._BN_SLICES, ](self._st, c, out) return out^ @@ -196,7 +196,7 @@ struct SetStream[patterns: List[String], allow_empty: Bool = False]( var out = List[SetMatch]() bitnfa_stream_close[ d=Self._bn, - pool=Self._db._SBN_POOL, - slices=Self._db._SBN_SLICES, + pool=Self._db._BN_POOL, + slices=Self._db._BN_SLICES, ](self._st, out) return out^ diff --git a/test/test_set_ac.mojo b/test/test_set_ac.mojo index d490f82..3e804ae 100644 --- a/test/test_set_ac.mojo +++ b/test/test_set_ac.mojo @@ -23,7 +23,6 @@ from emberregex.set_ac import ( AC_MAX, AC_POOL_CAP, AC_TABLE_CAP, - ac_cls_arr, ac_rep_arr, ac_scan, ac_table_str, @@ -67,7 +66,7 @@ def ac_direct_scan[ comptime A = build_ac(S.nfa, S.num_patterns, True) comptime V = ac_view(A) comptime T = static_bytes[ac_table_str[A.num_states * A.num_classes](A)]() - comptime C = ac_cls_arr(A) + comptime C = int_arr[DType.uint8, 256](A.class_map, 0) comptime R = ac_rep_arr[2 * A.num_states](A) comptime P = int_arr[DType.int32, len(A.pool)](A.pool, 0) return ac_scan[v=V, table=T, cls=C, rep=R, pool=P](input) diff --git a/test/test_set_bench_coverage.mojo b/test/test_set_bench_coverage.mojo index 663e626..f3662de 100644 --- a/test/test_set_bench_coverage.mojo +++ b/test/test_set_bench_coverage.mojo @@ -11,12 +11,11 @@ from emberregex.set_bitnfa import ( bitnfa_scan, build_bitnfa, ) -from emberregex.static_bytes import int_arr, list_arr, static_bytes +from emberregex.static_bytes import int_arr, list_arr, static_bytes, table_bytes from emberregex.set_dfa import ( build_multi_dfa, mdfa_scan, mdfa_slices_arr, - mdfa_table_str, ) from emberregex.set_nfa import build_union_nfa from emberregex.set_pike import set_pike_scan @@ -139,7 +138,9 @@ def mdfa_direct_scan[ """Mirror of the bench's phase-2 baseline helper.""" comptime S = RegexSet[patterns] comptime MD = build_multi_dfa(S.nfa, S.nfa.can_use_dfa) - comptime T = static_bytes[mdfa_table_str[MD.num_states * 256](MD)]() + comptime T = static_bytes[ + table_bytes[DType.int16](MD.table, MD.num_states * 256) + ]() comptime P = int_arr[DType.int32, len(MD.pool)](MD.pool, 0) comptime SL = mdfa_slices_arr[6 * MD.num_states](MD) return mdfa_scan[d=MD, table=T, pool=P, slices=SL](input) diff --git a/test/test_set_phase2.mojo b/test/test_set_phase2.mojo index 22f1ec2..51b638f 100644 --- a/test/test_set_phase2.mojo +++ b/test/test_set_phase2.mojo @@ -13,12 +13,11 @@ instead, which would silently stop exercising this engine. """ from emberregex import SetMatch, RegexSet -from emberregex.static_bytes import int_arr, static_bytes +from emberregex.static_bytes import int_arr, static_bytes, table_bytes from emberregex.set_dfa import ( build_multi_dfa, mdfa_scan, mdfa_slices_arr, - mdfa_table_str, MultiDFA, ) from emberregex.set_nfa import build_union_nfa @@ -33,7 +32,9 @@ def _mdfa_scan[ """Scan on the multi-accept DFA, bypassing engine selection.""" comptime S = RegexSet[patterns] comptime MD = build_multi_dfa(S.nfa, S.nfa.can_use_dfa) - comptime T = static_bytes[mdfa_table_str[MD.num_states * 256](MD)]() + comptime T = static_bytes[ + table_bytes[DType.int16](MD.table, MD.num_states * 256) + ]() comptime P = int_arr[DType.int32, len(MD.pool)](MD.pool, 0) comptime SL = mdfa_slices_arr[6 * MD.num_states](MD) return mdfa_scan[d=MD, table=T, pool=P, slices=SL](input) diff --git a/test/test_set_phase6.mojo b/test/test_set_phase6.mojo index 791615b..6826c74 100644 --- a/test/test_set_phase6.mojo +++ b/test/test_set_phase6.mojo @@ -88,13 +88,13 @@ def _bytes(s: String) -> List[Byte]: def test_stream_lane_available() raises: comptime S = RegexSet[["ERROR", "\\d+ms"]] - comptime can = S._can_stream + comptime can = S._bitnfa.valid assert_true(can) # Word boundaries keep a set off the automaton lanes entirely, so it # cannot stream; `SetStream` refuses such a set at COMPILE time # (comptime assert), which is why there is no runtime test here. comptime T = RegexSet[["\\bcat\\b"]] - comptime cant = T._can_stream + comptime cant = T._bitnfa.valid assert_false(cant) From 5c6f4b0cc1b7e8ef7047c840a60fd512bdc3a7f1 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 15:28:15 -0600 Subject: [PATCH 57/65] cleanup: drop the dead literal-chain walk budget extract_literal_chains expands each state at most once (the seen set), and an expansion pushes at most two successors, so the walk pops at most 2 * num_states + 1 entries; the 2 * num_states + 8 budget could never run out. walk_pops stays (test_set_ac pins it). Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/set_literal.mojo | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/emberregex/set_literal.mojo b/emberregex/set_literal.mojo index d8774ca..ec51d53 100644 --- a/emberregex/set_literal.mojo +++ b/emberregex/set_literal.mojo @@ -61,7 +61,7 @@ struct LiteralSet(Copyable, Movable): var walk_pops: Int """How many states the head walk popped. Diagnostic only — it exists so a test can pin that the walk is bounded by the NFA, not by the - entry cap (see the budget note in extract_literal_chains).""" + entry cap (see the visited-set note in extract_literal_chains).""" def __init__(out self): self.valid = False @@ -122,18 +122,11 @@ def extract_literal_chains( # rather than refused: a diamond in the epsilon region is legitimate # (it just yields a duplicate head), while a real cycle always puts a # two-way SPLIT on some chain, and the chain walk below refuses that. - # - # The budget survives only as a belt-and-braces bound; `seen` makes - # it unreachable. var heads = List[Int]() var stack: List[Int] = [nfa.start] var seen = List[Bool](fill=False, length=num_states) - var budget = 2 * num_states + 8 while len(stack) > 0: - budget -= 1 result.walk_pops += 1 - if budget < 0: - return result^ var s = stack.pop() if s < 0 or s >= num_states: return result^ From 7fa2351f7fe6c384391dacc762be57bddbe7d998 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 15:30:44 -0600 Subject: [PATCH 58/65] cleanup: stdlib sort for the bitnfa report ids (NEEDS MEASUREMENT) _emit_bits hand-rolled an insertion sort over the accepting ids before its dedup; use sort(ids). Runtime, once per accepting byte on the bitnfa block and stream lanes: revert if bench_set's bitnfa / stream rows regress (the ids lists are small and near-sorted, where insertion sort can win). Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/set_bitnfa.mojo | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/emberregex/set_bitnfa.mojo b/emberregex/set_bitnfa.mojo index c50d67c..358be52 100644 --- a/emberregex/set_bitnfa.mojo +++ b/emberregex/set_bitnfa.mojo @@ -586,13 +586,7 @@ def _emit_bits[ for i in range(n): ids.append(Int(pl.unsafe_get(off + i))) # Sort ascending and collapse duplicates. - for i in range(1, len(ids)): - var key = ids[i] - var j = i - 1 - while j >= 0 and ids[j] > key: - ids[j + 1] = ids[j] - j -= 1 - ids[j + 1] = key + sort(ids) for i in range(len(ids)): if i > 0 and ids[i] == ids[i - 1]: continue From 8805f072d7e384521008ba158350c3675d759f55 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 15:32:21 -0600 Subject: [PATCH 59/65] cleanup: stdlib sort in the set Pike report flush (NEEDS MEASUREMENT) _flush_reports hand-rolled an insertion sort over the ids reported at a position; use sort(ids). Runtime, once per position on the tagged Pike lane (word-boundary sets, residual Pike): revert if bench_set's word-boundary scan row regresses. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/set_pike.mojo | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/emberregex/set_pike.mojo b/emberregex/set_pike.mojo index 46d05d6..8bb06b5 100644 --- a/emberregex/set_pike.mojo +++ b/emberregex/set_pike.mojo @@ -276,13 +276,7 @@ def _flush_spans( def _flush_reports(mut ids: List[Int], end: Int, mut out: List[SetMatch]): """Emit this position's reports in ascending id order.""" - for i in range(1, len(ids)): - var key = ids[i] - var j = i - 1 - while j >= 0 and ids[j] > key: - ids[j + 1] = ids[j] - j -= 1 - ids[j + 1] = key + sort(ids) for i in range(len(ids)): out.append(SetMatch(ids[i], end)) From 6e2e1181fa1dd5539578e1a9a1be8a919b2da936 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 15:34:26 -0600 Subject: [PATCH 60/65] cleanup: stdlib sort in the SOM Pike span flush (NEEDS MEASUREMENT) _flush_spans insertion-sorted two parallel lists (ids, starts); emit the spans first and sort that slice of `out` by id with stdlib sort instead (ids are unique per position, so the order is unambiguous). Runtime, once per position on the SOM-carrying Pike (scan_som / scan_spans of word-boundary and reverse-cap-blowup sets): revert if those rows regress. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/set_pike.mojo | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/emberregex/set_pike.mojo b/emberregex/set_pike.mojo index 8bb06b5..0de305b 100644 --- a/emberregex/set_pike.mojo +++ b/emberregex/set_pike.mojo @@ -257,22 +257,20 @@ def _pike_add[ def _flush_spans( - mut ids: List[Int], mut som: List[Int], end: Int, mut out: List[SetSpan] + ids: List[Int], som: List[Int], end: Int, mut out: List[SetSpan] ): - """Emit this position's reports in ascending id order.""" - for i in range(1, len(ids)): - var key = ids[i] - var key_som = som[i] - var j = i - 1 - while j >= 0 and ids[j] > key: - ids[j + 1] = ids[j] - som[j + 1] = som[j] - j -= 1 - ids[j + 1] = key - som[j + 1] = key_som + """Emit this position's reports in ascending id order (ids are unique + per position: one MATCH state per pattern, visited once).""" + var base = len(out) for i in range(len(ids)): out.append(SetSpan(ids[i], som[i], end)) + @always_inline + def by_id(a: SetSpan, b: SetSpan) -> Bool: + return a.id < b.id + + sort(Span(out)[base:], by_id) + def _flush_reports(mut ids: List[Int], end: Int, mut out: List[SetMatch]): """Emit this position's reports in ascending id order.""" From a4f3b6e1d1a6640001bc37f286565538d6cd2bea Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 15:35:58 -0600 Subject: [PATCH 61/65] cleanup: stdlib sort in leftmost_nonoverlapping (NEEDS MEASUREMENT) Both hand-rolled insertion sorts (per-id start/longest order, then the cross-id start/id order) become sort(list, cmp). Both orders are total over their inputs, so stability is moot. Runtime, once per scan_spans call: revert if the scan_spans rows regress. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/set_reverse.mojo | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/emberregex/set_reverse.mojo b/emberregex/set_reverse.mojo index 9cc5d93..dac7d77 100644 --- a/emberregex/set_reverse.mojo +++ b/emberregex/set_reverse.mojo @@ -838,6 +838,17 @@ def leftmost_nonoverlapping( Output is ordered by (start, id). """ + + # Both orders are total over their inputs: one id reports each end + # once, and one id's survivors have strictly increasing starts. + @always_inline + def start_then_longest(a: SetSpan, b: SetSpan) -> Bool: + return a.start < b.start or (a.start == b.start and a.end > b.end) + + @always_inline + def start_then_id(a: SetSpan, b: SetSpan) -> Bool: + return a.start < b.start or (a.start == b.start and a.id < b.id) + var out = List[SetSpan]() for id in range(num_patterns): # (start asc, end desc) so the first survivor at a start is the @@ -846,16 +857,7 @@ def leftmost_nonoverlapping( for s in spans: if s.id == id and s.start >= 0: mine.append(s) - for i in range(1, len(mine)): - var key = mine[i] - var j = i - 1 - while j >= 0 and ( - mine[j].start > key.start - or (mine[j].start == key.start and mine[j].end < key.end) - ): - mine[j + 1] = mine[j] - j -= 1 - mine[j + 1] = key + sort(mine, start_then_longest) var next_allowed = 0 for s in mine: if s.start < next_allowed: @@ -864,16 +866,7 @@ def leftmost_nonoverlapping( # An empty match must still advance, or iteration stalls. next_allowed = s.end if s.end > s.start else s.start + 1 # (start, id) order across ids. - for i in range(1, len(out)): - var key = out[i] - var j = i - 1 - while j >= 0 and ( - out[j].start > key.start - or (out[j].start == key.start and out[j].id > key.id) - ): - out[j + 1] = out[j] - j -= 1 - out[j + 1] = key + sort(out, start_then_id) return out^ From 85f2318763855041f354906b7e5fc1560c4976fb Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 15:41:22 -0600 Subject: [PATCH 62/65] cleanup: mojo format (pre-existing blank-line drift in static_dfa.mojo) Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/static_dfa.mojo | 3 --- 1 file changed, 3 deletions(-) diff --git a/emberregex/static_dfa.mojo b/emberregex/static_dfa.mojo index 89c5105..60cd55f 100644 --- a/emberregex/static_dfa.mojo +++ b/emberregex/static_dfa.mojo @@ -1844,9 +1844,6 @@ def _find_exit2[ return input_len - - - @always_inline def _edfa_accel_skip[ origin: Origin, //, d: EagerDFA From ad8394bdcba1d4d3e03f6589d186b7f176e2c702 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 16:54:39 -0600 Subject: [PATCH 63/65] cleanup: flags ride into _build_static_nfa as an operand, not through _pat Routing Regex.nfa through the _pat ternary put that expression into every lane method's type (via _num_slots) and cost ~20% compile time on bench.mojo. nfa = _build_static_nfa(pattern, flags.value) keeps the default-flags instantiations' expression the same shape as before; _pat stays only at the backtracker call sites, where it measured free. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/engine.mojo | 58 +++++++++--------------------------------- emberregex/nfa.mojo | 43 +++++++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 48 deletions(-) diff --git a/emberregex/engine.mojo b/emberregex/engine.mojo index f402c00..efdebd2 100644 --- a/emberregex/engine.mojo +++ b/emberregex/engine.mojo @@ -15,15 +15,13 @@ backtrack.mojo for what that does and does not flatten. from .constants import ( CHAR_BACKSLASH, - CHAR_LPAREN, CHAR_NEWLINE, CHAR_NINE, CHAR_ONE, - CHAR_RPAREN, - CHAR_STAR, CHAR_ZERO, ) from .nfa import ( + apply_flags, _build_static_nfa, _nfa_has_backref, split_cycle_flags, @@ -1155,41 +1153,6 @@ struct _LFWalk[num_slots: Int, span: Bool, origin: MutOrigin]( self.pike = pike -def _apply_flags(pattern: String, flags: RegexFlags) -> String: - """`pattern` with `flags` spelled as one leading inline group - (`(?imsxu)`), placed after any leading `(*UTF8)` verbs — the parser - only accepts those first (see `Parser._consume_verbs`).""" - var letters = String() - if flags.ignorecase(): - letters += "i" - if flags.multiline(): - letters += "m" - if flags.dotall(): - letters += "s" - if flags.verbose(): - letters += "x" - if flags.unicode(): - letters += "u" - var b = pattern.as_bytes() - var pos = 0 - while ( - pos + 2 < len(b) and b[pos] == CHAR_LPAREN and b[pos + 1] == CHAR_STAR - ): - var close = pos + 2 - while close < len(b) and b[close] != CHAR_RPAREN: - close += 1 - if close >= len(b): - break - pos = close + 1 - return ( - String(unsafe_from_utf8=b[:pos]) - + "(?" - + letters - + ")" - + String(unsafe_from_utf8=b[pos:]) - ) - - struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( Copyable, Movable ): @@ -1206,15 +1169,18 @@ struct Regex[pattern: String, flags: RegexFlags = RegexFlags()]( recurse — see backtrack.mojo. """ - # The pattern every lane compiles: exactly `pattern` for the default - # flags, so those instantiations share the memoized NFA and the - # backtracker's symbol names with the flag-free spelling. The ternary - # is load-bearing: a String RETURNED by a comptime call is a different - # interpreter value (and symbol mangling) even when equal. - comptime _pat = Self.pattern if Self.flags.value == 0 else _apply_flags( - Self.pattern, Self.flags + # The pattern the backtracker re-derives its NFA from: exactly + # `pattern` for the default flags, so its per-state symbols and NFA + # memo entry are the flag-free ones. The ternary is load-bearing: a + # String RETURNED by a comptime call is a different interpreter value + # (and symbol mangling) even when equal. `nfa` takes the flags as an + # operand instead of going through `_pat`: every lane method's type + # mentions `nfa` (via `_num_slots`), and routing it through the + # ternary cost ~20% of compile time on bench.mojo. + comptime _pat = Self.pattern if Self.flags.value == 0 else apply_flags( + Self.pattern, Self.flags.value ) - comptime nfa = _build_static_nfa(Self._pat) + comptime nfa = _build_static_nfa(Self.pattern, Self.flags.value) # One Tarjan pass and one depth plan per pattern: the selection # predicates below used to each call `split_cycle_flags(Self.nfa)` # inside their own bodies, and calls made inside interpreted bodies diff --git a/emberregex/nfa.mojo b/emberregex/nfa.mojo index ddcbe1e..2b906ac 100644 --- a/emberregex/nfa.mojo +++ b/emberregex/nfa.mojo @@ -10,6 +10,9 @@ from std.math import max, min from .constants import ( CHAR_A_LOWER, CHAR_A_UPPER, + CHAR_LPAREN, + CHAR_RPAREN, + CHAR_STAR, CHAR_Z_LOWER, CHAR_Z_UPPER, ascii_to_lower, @@ -1464,7 +1467,43 @@ def _add_case_folding(mut cs: CharSet): cs.ranges.extend(new_ranges^) -def _build_static_nfa(pattern: String) -> NFA: +def apply_flags(pattern: String, flag_bits: Int) -> String: + """`pattern` with `flags` spelled as one leading inline group + (`(?imsxu)`), placed after any leading `(*UTF8)` verbs — the parser + only accepts those first (see `Parser._consume_verbs`).""" + var flags = RegexFlags(flag_bits) + var letters = String() + if flags.ignorecase(): + letters += "i" + if flags.multiline(): + letters += "m" + if flags.dotall(): + letters += "s" + if flags.verbose(): + letters += "x" + if flags.unicode(): + letters += "u" + var b = pattern.as_bytes() + var pos = 0 + while ( + pos + 2 < len(b) and b[pos] == CHAR_LPAREN and b[pos + 1] == CHAR_STAR + ): + var close = pos + 2 + while close < len(b) and b[close] != CHAR_RPAREN: + close += 1 + if close >= len(b): + break + pos = close + 1 + return ( + String(unsafe_from_utf8=b[:pos]) + + "(?" + + letters + + ")" + + String(unsafe_from_utf8=b[pos:]) + ) + + +def _build_static_nfa(pattern: String, flags: Int = 0) -> NFA: """Parse and build NFA — called at compile time. Aborts on invalid pattern (produces compile error at comptime). @@ -1477,7 +1516,7 @@ def _build_static_nfa(pattern: String) -> NFA: states, and a linker failure past a few hundred). """ try: - var ast = parse(pattern) + var ast = parse(pattern if flags == 0 else apply_flags(pattern, flags)) var merged_flags = ast.flags return build_nfa(ast^, merged_flags) except e: From d6186cbb6d41f5a0d4df4ec3fda1014073574c52 Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 16:54:39 -0600 Subject: [PATCH 64/65] Revert "cleanup: stdlib sort in the set Pike report flush (NEEDS MEASUREMENT)" This reverts commit 8805f072d7e384521008ba158350c3675d759f55. Co-Authored-By: Claude Opus 5.5 (1M context) --- emberregex/set_pike.mojo | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/emberregex/set_pike.mojo b/emberregex/set_pike.mojo index 0de305b..8297afd 100644 --- a/emberregex/set_pike.mojo +++ b/emberregex/set_pike.mojo @@ -274,7 +274,13 @@ def _flush_spans( def _flush_reports(mut ids: List[Int], end: Int, mut out: List[SetMatch]): """Emit this position's reports in ascending id order.""" - sort(ids) + for i in range(1, len(ids)): + var key = ids[i] + var j = i - 1 + while j >= 0 and ids[j] > key: + ids[j + 1] = ids[j] + j -= 1 + ids[j + 1] = key for i in range(len(ids)): out.append(SetMatch(ids[i], end)) From 239c742d249e32e9c90f57357a1dc4ca9b5f972a Mon Sep 17 00:00:00 2001 From: Brian Grenier Date: Wed, 23 Sep 2026 17:24:12 -0600 Subject: [PATCH 65/65] test: cover apply_flags at runtime apply_flags only ran in the comptime interpreter, so the coverage gate counted its 26 runtime lines as uncovered (98.65% < 99.01% baseline). Three runtime asserts on the existing flags test bring it back to 99.03%. Co-Authored-By: Claude Opus 5.5 (1M context) --- test/test_flags.mojo | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/test_flags.mojo b/test/test_flags.mojo index 58b0468..f40361e 100644 --- a/test/test_flags.mojo +++ b/test/test_flags.mojo @@ -1,6 +1,7 @@ """Tests for inline flags: (?i) ignorecase, (?m) multiline, (?s) dotall.""" from emberregex import Regex, RegexFlags +from emberregex.nfa import apply_flags from std.testing import assert_true, assert_false, assert_equal, TestSuite @@ -127,6 +128,18 @@ def test_explicit_flags_param() raises: var r = re2.search("foo\nHELLO") assert_true(r.matched) assert_equal(r.span(), inline2.search("foo\nHELLO").span()) + # The same rewrite at runtime (the comptime asserts above run it only + # in the interpreter): every letter, a verb prefix, an unclosed verb. + comptime ALL = ( + RegexFlags.IGNORECASE + | RegexFlags.MULTILINE + | RegexFlags.DOTALL + | RegexFlags.VERBOSE + | RegexFlags.UNICODE + ) + assert_equal(apply_flags("ab", ALL), "(?imsxu)ab") + assert_equal(apply_flags("(*UTF8)ab", RegexFlags.DOTALL), "(*UTF8)(?s)ab") + assert_equal(apply_flags("(*UTF8", RegexFlags.MULTILINE), "(?m)(*UTF8") def test_combined_all_three() raises: