diff --git a/src/main/java/com/williamcallahan/javachat/service/HtmlContentExtractor.java b/src/main/java/com/williamcallahan/javachat/service/HtmlContentExtractor.java index 5ad154be..b6a10ceb 100644 --- a/src/main/java/com/williamcallahan/javachat/service/HtmlContentExtractor.java +++ b/src/main/java/com/williamcallahan/javachat/service/HtmlContentExtractor.java @@ -222,13 +222,16 @@ private void appendTableContent(Element table, StringBuilder sb) { */ private boolean isNavigationElement(Element element) { String className = AsciiTextNormalizer.toLowerAscii(element.className()); + Set classTokens = element.classNames().stream() + .map(AsciiTextNormalizer::toLowerAscii) + .collect(Collectors.toSet()); String id = AsciiTextNormalizer.toLowerAscii(element.id()); String text = AsciiTextNormalizer.toLowerAscii(element.text()); return className.contains("nav") || className.contains("menu") || className.contains("sidebar") - || className.contains("header") + || classTokens.contains("header") || className.contains("footer") || id.contains("nav") || id.contains("menu") diff --git a/src/test/java/com/williamcallahan/javachat/service/HtmlContentExtractorTest.java b/src/test/java/com/williamcallahan/javachat/service/HtmlContentExtractorTest.java index e8b0b20c..d032779a 100644 --- a/src/test/java/com/williamcallahan/javachat/service/HtmlContentExtractorTest.java +++ b/src/test/java/com/williamcallahan/javachat/service/HtmlContentExtractorTest.java @@ -267,4 +267,58 @@ void retainsPackagePagesAsUnanchoredJavaApiOverviews() { assertTrue(extraction.anchoredSections().isEmpty()); assertTrue(document.selectFirst("nav") != null, "Extraction must use a clone before removing navigation"); } + + @Test + void pageHeaderWrapperRetainsNestedHeading() { + Document document = Jsoup.parse(""" +
+

Important Page Title

+

Body text that should always remain.

+
+ """); + HtmlContentExtractor extractor = new HtmlContentExtractor(); + + String extractedText = extractor.extractCleanContent(document); + + assertTrue(extractedText.contains("Important Page Title"), "nested h1 in page-header must survive"); + assertTrue(extractedText.contains("Body text that should always remain."), "sibling body text must survive"); + } + + @Test + void exactHeaderClassTokenIsStillRemoved() { + Document document = Jsoup.parse(""" +
+

Site Header Title

+

Body text that should always remain.

+
+ """); + HtmlContentExtractor extractor = new HtmlContentExtractor(); + + String extractedText = extractor.extractCleanContent(document); + + assertFalse(extractedText.contains("Site Header Title"), "exact header class token must still be removed"); + assertTrue(extractedText.contains("Body text that should always remain."), "sibling body text must survive"); + } + + @Test + void compositeNavClassesAreStillRemoved() { + Document document = Jsoup.parse(""" +
+
Home
+
About
+
Docs
+
Blog
+

Body text that should always remain.

+
+ """); + HtmlContentExtractor extractor = new HtmlContentExtractor(); + + String extractedText = extractor.extractCleanContent(document); + + assertFalse(extractedText.contains("Home"), "top-nav must still be removed"); + assertFalse(extractedText.contains("About"), "main-nav must still be removed"); + assertFalse(extractedText.contains("Docs"), "header-nav must still be removed"); + assertFalse(extractedText.contains("Blog"), "primary-nav must still be removed"); + assertTrue(extractedText.contains("Body text that should always remain."), "body text must survive"); + } }