Handle empty and FTS5-operator search queries without raising a 500

app/helpers/searches_helper.rb

@@ -23,6 +23,6 @@ def highlight_searched_content(leaf, content, query)
 
   private
     def whole_word_matchers(terms)
-      terms.map { |term| /\b#{term}\b/ }
+      terms.map { |term| /\b#{Regexp.escape(term)}\b/ }
     end
 end

app/models/leaf/searchable.rb

@@ -15,9 +15,9 @@ def reindex_all
     end
 
     def sanitize_query_syntax(terms)
-      terms = terms.to_s
+      terms = terms.to_s.scrub
       terms = remove_invalid_search_characters(terms)
-      terms = remove_unbalanced_quotes(terms)
+      terms = quote_query_tokens(terms)
       terms.presence
     end
 
@@ -50,6 +50,8 @@ def matches_for_highlight(terms)
         .pick(Arel.sql("highlight(leaf_search_index, 1, '<mark>', '</mark>')"))
 
       content ? unique_matching_terms(content) : []
+    else
+      []
     end
   end
 
@@ -106,12 +108,22 @@ def remove_invalid_search_characters(terms)
           terms.gsub(/[^\w"]/, " ")
         end
 
-        def remove_unbalanced_quotes(terms)
-          if terms.count("\"").even?
-            terms
-          else
-            terms.gsub("\"", " ")
-          end
+        # After stripping the characters FTS5 can't tokenize, the remaining
+        # input may still be an FTS5 boolean operator (AND/OR/NOT/NEAR) or
+        # carry an unbalanced double quote — either of which makes SQLite raise
+        # a syntax error. Rebuild the query from its balanced "quoted phrases"
+        # and bare words, wrapping every token as a quoted string literal so
+        # arbitrary input is matched literally instead of parsed as syntax.
+        #
+        # Match empty quote pairs too (`[^"]*`, not `+`) so a stray `""` is
+        # consumed in place rather than pairing its closing quote with the next
+        # opening one — which would shift the boundaries of a following phrase
+        # and split it into separate word matches. Empty tokens are then dropped.
+        def quote_query_tokens(terms)
+          terms.scan(/"[^"]*"|\w+/)
+            .filter_map { |token| token.delete('"').presence }
+            .map { |token| %("#{token}") }
+            .join(" ")
         end
     end
 end

test/controllers/books/searches_controller_test.rb

@@ -41,6 +41,21 @@ class Books::SearchesControllerTest < ActionDispatch::IntegrationTest
     assert_select "p", text: /no matches/i
   end
 
+  test "create shows no matches when the search uses FTS5 operator syntax" do
+    [ "OR", "AND", "NOT", "great OR", "NEAR handbook", "great AND NOT" ].each do |query|
+      post book_search_path(books(:handbook)), params: { search: query }
+
+      assert_response :success, "expected #{query.inspect} to render without error"
+    end
+  end
+
+  test "create still finds matches for an ordinary multi-word query" do
+    post book_search_path(books(:handbook)), params: { search: "great handbook" }
+
+    assert_response :success
+    assert_select "a.search__result"
+  end
+
   test "create does not find trashed pages" do
     leaves(:summary_page).trashed!
 

test/controllers/leafables_controller_test.rb

@@ -30,6 +30,44 @@ class LeafablesControllerTest < ActionDispatch::IntegrationTest
     assert_select "mark", "great"
   end
 
+  test "show does not raise when the search query sanitizes to empty" do
+    sign_out
+    books(:handbook).update!(published: true)
+    Leaf.reindex_all
+
+    [ "^$", "!!!", "🙂", "\"" ].each do |query|
+      get leafable_slug_path(leaves(:welcome_page)), params: { search: query }
+
+      assert_response :success, "expected #{query.inspect} to render without error"
+      assert_in_body "a great handbook."
+    end
+  end
+
+  test "show does not raise when the search query uses FTS5 operator syntax" do
+    sign_out
+    books(:handbook).update!(published: true)
+    Leaf.reindex_all
+
+    [ "OR", "AND", "NOT", "great OR", "NEAR handbook", "great AND NOT" ].each do |query|
+      get leafable_slug_path(leaves(:welcome_page)), params: { search: query }
+
+      assert_response :success, "expected #{query.inspect} to render without error"
+    end
+  end
+
+  test "show does not raise when a phrase match spans regex metacharacters" do
+    sign_out
+    books(:handbook).update!(published: true)
+
+    sections(:welcome).update!(body: "alpha(beta gamma in the body")
+    leaves(:welcome_section).reindex
+
+    get leafable_slug_path(leaves(:welcome_section)), params: { search: "alpha_beta" }
+
+    assert_response :success
+    assert_select "mark", text: /alpha\(beta/
+  end
+
   test "show does not allow public access to an unpublished book" do
     sign_out
 

test/helpers/searches_helper_test.rb

@@ -1,6 +1,8 @@
 require "test_helper"
 
 class SearchesHelperTest < ActionView::TestCase
+  include PagesHelper
+
   test "sanitize_search_result preserves mark tags" do
     assert_equal "<mark>findme</mark> text", sanitize_search_result("<mark>findme</mark> text")
   end
@@ -16,4 +18,14 @@ class SearchesHelperTest < ActionView::TestCase
   test "sanitize_search_result strips attributes from mark tags" do
     assert_equal "<mark>findme</mark> text", sanitize_search_result('<mark class="hidden">findme</mark> text')
   end
+
+  test "highlight_searched_content handles matched terms containing regex metacharacters" do
+    leaf = Struct.new(:terms) do
+      def matches_for_highlight(_query) = terms
+    end.new([ "alpha(beta" ])
+
+    result = highlight_searched_content(leaf, "alpha(beta in the body", "alpha beta")
+
+    assert_includes result, "<mark>alpha(beta</mark>"
+  end
 end

test/models/leaf/searchable_test.rb

@@ -38,6 +38,41 @@ class Leaf::SearchableTest < ActiveSupport::TestCase
     assert_empty markup
   end
 
+  test "matches_for_highlight is empty when the query sanitizes to nothing" do
+    assert_empty leaves(:welcome_page).matches_for_highlight("^$")
+    assert_empty leaves(:welcome_page).matches_for_highlight("🙂")
+    assert_empty leaves(:welcome_page).matches_for_highlight("\"")
+  end
+
+  test "search treats FTS5 operators as literal terms rather than syntax" do
+    assert_empty Leaf.search("OR")
+    assert_empty Leaf.search("great AND NOT")
+    assert_empty Leaf.search("great OR handbook")
+
+    assert_includes Leaf.search("great handbook"), leaves(:welcome_page)
+    assert_includes Leaf.search("\"great handbook\""), leaves(:welcome_page)
+  end
+
+  test "a stray empty quote pair does not split a following phrase" do
+    sections(:welcome).update!(body: "great old handbook")
+    leaves(:welcome_section).reindex
+
+    results = Leaf.search("\"\" \"great handbook\"")
+
+    assert_includes results, leaves(:welcome_page)
+    assert_not_includes results, leaves(:welcome_section)
+  end
+
+  test "search does not raise on invalid UTF-8 byte sequences" do
+    malformed = "caf\xFF".dup.force_encoding("UTF-8")
+    assert_not malformed.valid_encoding?
+
+    assert_nothing_raised do
+      assert_empty Leaf.search(malformed)
+      assert_empty leaves(:welcome_page).matches_for_highlight(malformed)
+    end
+  end
+
   test "indexing sanitizes section body" do
     section = Section.new(body: 'findme Tom & Jerry <img src=x onerror="alert(1)">')
     books(:handbook).press(section, title: "Safe Title")