refactor(indexing): parse Markdown front matter with SnakeYAML

build.gradle.kts

@@ -66,6 +66,7 @@ val nativeImageBuildArgs =
     listOf(
         "--no-fallback",
         "-H:+ReportExceptionStackTraces",
+        "-H:IncludeResources=shows-markdown/.*\\.md$",
         "--initialize-at-build-time=io.opentelemetry.api",
         "--initialize-at-build-time=io.opentelemetry.context",
         "--initialize-at-build-time=io.opentelemetry.instrumentation.api",

src/main/java/org/apache/solr/mcp/server/indexing/documentcreator/MarkdownDocumentCreator.java

@@ -22,20 +22,28 @@
 import java.util.ArrayList;
 import java.util.HexFormat;
 import java.util.List;
+import java.util.Map;
 import org.apache.solr.common.SolrInputDocument;
 import org.commonmark.Extension;
+import org.commonmark.ext.front.matter.YamlFrontMatterBlock;
 import org.commonmark.ext.front.matter.YamlFrontMatterExtension;
-import org.commonmark.ext.front.matter.YamlFrontMatterVisitor;
 import org.commonmark.node.AbstractVisitor;
 import org.commonmark.node.Code;
-import org.commonmark.node.CustomBlock;
 import org.commonmark.node.Heading;
 import org.commonmark.node.Node;
+import org.commonmark.node.SourceSpan;
 import org.commonmark.node.Text;
+import org.commonmark.parser.IncludeSourceSpans;
 import org.commonmark.parser.Parser;
 import org.commonmark.renderer.text.TextContentRenderer;
 import org.jspecify.annotations.Nullable;
 import org.springframework.stereotype.Component;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.LoaderOptions;
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.constructor.SafeConstructor;
+import org.yaml.snakeyaml.representer.Representer;
+import org.yaml.snakeyaml.resolver.Resolver;
 
 /**
  * Utility class for processing markdown documents and converting them to
@@ -108,9 +116,21 @@ public class MarkdownDocumentCreator implements SolrDocumentCreator {
 
 	private final TextContentRenderer textContentRenderer;
 
+	private final Yaml yaml;
+
 	public MarkdownDocumentCreator() {
 		List<Extension> extensions = List.of(YamlFrontMatterExtension.create());
-		this.parser = Parser.builder().extensions(extensions).build();
+		// Source spans let the front matter block be sliced out verbatim for SnakeYAML
+		this.parser = Parser.builder().extensions(extensions).includeSourceSpans(IncludeSourceSpans.BLOCKS).build();
+		// No implicit resolvers: every scalar stays the text as written (2026-01-01
+		// is not a Date, 8.4 is not a Double); Solr's schema guessing types them.
+		LoaderOptions options = new LoaderOptions();
+		this.yaml = new Yaml(new SafeConstructor(options), new Representer(new DumperOptions()), new DumperOptions(),
+				options, new Resolver() {
+					@Override
+					protected void addImplicitResolvers() {
+					}
+				});
 		this.textContentRenderer = TextContentRenderer.builder().build();
 	}
 
@@ -150,7 +170,7 @@ public List<SolrInputDocument> create(String markdown) throws DocumentProcessing
 
 		SolrInputDocument doc = new SolrInputDocument();
 
-		addFrontMatterFields(document, doc);
+		addFrontMatterFields(markdown, document, doc);
 
 		// Solr's default schema requires a unique key. A content-derived id keeps
 		// re-indexing of the same markdown idempotent (same input, same document)
@@ -178,55 +198,49 @@ public List<SolrInputDocument> create(String markdown) throws DocumentProcessing
 	}
 
 	/**
-	 * Extracts YAML front matter entries into document fields and unlinks the front
-	 * matter block so it is excluded from the rendered body content.
+	 * Parses the YAML front matter with SnakeYAML and adds each entry as a field:
+	 * scalars as their text, sequences as multi-valued fields, nested mappings
+	 * flattened with underscores. The block is then unlinked so the rendered body
+	 * contains only the document text.
 	 */
-	private void addFrontMatterFields(Node document, SolrInputDocument doc) {
-		YamlFrontMatterVisitor frontMatterVisitor = new YamlFrontMatterVisitor();
-		document.accept(frontMatterVisitor);
-
-		frontMatterVisitor.getData().forEach((key, values) -> {
-			String fieldName = FieldNameSanitizer.sanitizeFieldName(key);
-			for (String value : flattenFlowSequences(values)) {
-				if (!value.isEmpty()) {
-					doc.addField(fieldName, value);
-				}
-			}
-		});
-
-		// The front matter block is metadata, not body text: remove it so the
-		// TextContentRenderer output contains only the document body
+	private void addFrontMatterFields(String markdown, Node document, SolrInputDocument doc) {
 		Node firstChild = document.getFirstChild();
-		if (firstChild instanceof CustomBlock) {
-			firstChild.unlink();
+		if (!(firstChild instanceof YamlFrontMatterBlock block)) {
+			return;
+		}
+		List<String> lines = new ArrayList<>();
+		for (SourceSpan span : block.getSourceSpans()) {
+			lines.add(markdown.substring(span.getInputIndex(), span.getInputIndex() + span.getLength()));
 		}
+		// The first and last spans are the --- delimiters
+		String text = String.join("\n", lines.subList(1, Math.max(1, lines.size() - 1)));
+		Object data;
+		try {
+			data = yaml.load(text);
+		} catch (RuntimeException e) {
+			throw new DocumentProcessingException("Failed to parse YAML front matter", e);
+		}
+		if (data instanceof Map<?, ?> entries) {
+			entries.forEach(
+					(key, value) -> addValue(doc, FieldNameSanitizer.sanitizeFieldName(String.valueOf(key)), value));
+		}
+		block.unlink();
 	}
 
-	/**
-	 * Expands simple YAML flow sequences into individual values.
-	 *
-	 * <p>
-	 * The CommonMark front matter extension parses block-style lists
-	 * ({@code - item}) into multiple values but passes flow-style lists
-	 * ({@code [a, b, c]}) through as a single literal string. Flow style is common
-	 * for tags in real-world markdown (Jekyll, Hugo), so split it here to produce
-	 * the same multi-valued field either way. Values containing commas inside
-	 * quotes are not supported and are kept as-is.
-	 */
-	private static List<String> flattenFlowSequences(List<String> values) {
-		List<String> result = new ArrayList<>(values.size());
-		for (String value : values) {
-			String trimmed = value.trim();
-			if (trimmed.length() >= 2 && trimmed.startsWith("[") && trimmed.endsWith("]") && !trimmed.contains("\"")
-					&& !trimmed.contains("'")) {
-				for (String element : trimmed.substring(1, trimmed.length() - 1).split(",")) {
-					result.add(element.trim());
+	private static void addValue(SolrInputDocument doc, String fieldName, Object value) {
+		switch (value) {
+			case null -> {
+			}
+			case Map<?, ?> nested -> nested.forEach(
+					(key, inner) -> addValue(doc, FieldNameSanitizer.sanitizeFieldName(fieldName + "_" + key), inner));
+			case Iterable<?> values -> values.forEach(element -> addValue(doc, fieldName, element));
+			default -> {
+				String text = String.valueOf(value);
+				if (!text.isEmpty()) {
+					doc.addField(fieldName, text);
 				}
-			} else {
-				result.add(value);
 			}
 		}
-		return result;
 	}
 
 	private static String contentHash(String markdown) {

src/test/java/org/apache/solr/mcp/server/indexing/ShowsSampleDataTest.java

@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.mcp.server.indexing;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.TreeMap;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.mcp.server.indexing.documentcreator.JsonDocumentCreator;
+import org.apache.solr.mcp.server.indexing.documentcreator.MarkdownDocumentCreator;
+import org.junit.jupiter.api.Test;
+
+/**
+ * The {@code shows} sample dataset ships in every format the indexing tools
+ * accept: {@code shows.json}, {@code shows.csv}, {@code shows.xml} and one
+ * Markdown file per show under {@code shows-markdown/}. These tests pin that
+ * the representations the server parses itself (JSON and Markdown) yield the
+ * same 61 documents; CSV and XML are forwarded to Solr and checked end to end
+ * in {@code ShowsSampleDataIntegrationTest}.
+ *
+ * <p>
+ * Two representation choices are worth knowing:
+ * <ul>
+ * <li>CSV carries multi-valued fields as <em>repeated column headers</em>
+ * ({@code genres,genres,genres}); Solr's CSV handler adds one value per
+ * non-empty cell under the same field name.</li>
+ * <li>XML is Solr's own update format ({@code <add><doc><field name=...>}); it
+ * is forwarded to Solr rather than parsed here, so its equality with the JSON
+ * documents is checked end to end in
+ * {@code ShowsSampleDataIntegrationTest}.</li>
+ * </ul>
+ */
+class ShowsSampleDataTest {
+
+	private static final int SHOWS = 61;
+
+	private final JsonDocumentCreator json = new JsonDocumentCreator(new ObjectMapper());
+
+	@Test
+	void jsonHas61ShowsWithUniqueIds() throws Exception {
+		Map<String, Map<String, List<String>>> shows = byId(json.create(resource("/shows.json")), "id");
+
+		assertThat(shows).hasSize(SHOWS);
+		assertThat(shows.get("netflix-001")).containsEntry("title", List.of("Stranger Things")).containsEntry("genres",
+				List.of("Sci-Fi", "Horror", "Drama"));
+	}
+
+	@Test
+	void markdownFilesParseToTheSameDocumentsAsJson() throws Exception {
+		Map<String, Map<String, List<String>>> expected = byId(json.create(resource("/shows.json")), "id");
+		MarkdownDocumentCreator markdown = new MarkdownDocumentCreator();
+
+		for (Map.Entry<String, Map<String, List<String>>> show : expected.entrySet()) {
+			List<SolrInputDocument> docs = markdown.create(resource("/shows-markdown/" + show.getKey() + ".md"));
+			assertThat(docs).as(show.getKey()).hasSize(1);
+			Map<String, List<String>> fields = fields(docs.getFirst(), "");
+
+			// The description is the document body, so it comes back as content
+			// (with the title heading) rather than as a front matter field.
+			Map<String, List<String>> frontMatter = new TreeMap<>(show.getValue());
+			List<String> description = frontMatter.remove("description");
+			assertThat(fields.remove("content")).as("%s content", show.getKey()).hasSize(1).first().asString()
+					.contains(description.getFirst());
+			assertThat(fields.remove("headings")).as("%s headings", show.getKey())
+					.isEqualTo(show.getValue().get("title"));
+			assertThat(fields).as(show.getKey()).containsExactlyEntriesOf(frontMatter);
+		}
+	}
+
+	private static Map<String, Map<String, List<String>>> byId(List<SolrInputDocument> docs, String idField) {
+		Map<String, Map<String, List<String>>> byId = new TreeMap<>();
+		for (SolrInputDocument doc : docs) {
+			Map<String, List<String>> fields = fields(doc, "");
+			String id = Objects.requireNonNull(fields.get(idField), "missing " + idField).getFirst();
+			assertThat(byId.put(id, fields)).as("duplicate id %s", id).isNull();
+		}
+		return byId;
+	}
+
+	/**
+	 * Field name to string values, with {@code prefix} stripped from every name.
+	 */
+	private static Map<String, List<String>> fields(SolrInputDocument doc, String prefix) {
+		Map<String, List<String>> fields = new TreeMap<>();
+		for (String name : doc.getFieldNames()) {
+			String key = name.startsWith(prefix) ? name.substring(prefix.length()) : name;
+			fields.put(key, doc.getFieldValues(name).stream().map(String::valueOf).toList());
+		}
+		return new LinkedHashMap<>(fields);
+	}
+
+	private static String resource(String path) throws IOException {
+		try (InputStream in = Objects.requireNonNull(ShowsSampleDataTest.class.getResourceAsStream(path),
+				"missing test resource " + path)) {
+			return new String(in.readAllBytes(), StandardCharsets.UTF_8);
+		}
+	}
+}

src/test/java/org/apache/solr/mcp/server/indexing/documentcreator/MarkdownDocumentCreatorTest.java

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.mcp.server.indexing.documentcreator;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import org.apache.solr.common.SolrInputDocument;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Front matter is YAML, so it is parsed by a YAML parser. These cases are the
+ * ones a line-by-line reader gets wrong: quoted scalars containing the
+ * delimiters, flow and block sequences, and nested mappings. Scalars stay the
+ * text as written; Solr's schema guessing types them.
+ */
+class MarkdownDocumentCreatorTest {
+
+	private final MarkdownDocumentCreator creator = new MarkdownDocumentCreator();
+
+	@Test
+	void frontMatterIsParsedAsYaml() {
+		SolrInputDocument doc = creator.create("""
+				---
+				id: show-1
+				title: "Star Wars: Andor, Season 2"
+				genres: [Sci-Fi, "Drama, Political"]
+				cast:
+				  - Diego Luna
+				  - Stellan Skarsgård
+				ratings:
+				  imdb: 8.4
+				  age: TV-14
+				seasons: 2
+				ongoing: false
+				released: 2022-09-21
+				empty:
+				---
+				# Andor
+
+				Prequel to Rogue One.
+				""").getFirst();
+
+		assertThat(doc.getFieldValue("id")).isEqualTo("show-1");
+		assertThat(doc.getFieldValue("title")).isEqualTo("Star Wars: Andor, Season 2");
+		assertThat(doc.getFieldValues("genres")).containsExactly("Sci-Fi", "Drama, Political");
+		assertThat(doc.getFieldValues("cast")).containsExactly("Diego Luna", "Stellan Skarsgård");
+		assertThat(doc.getFieldValue("ratings_imdb")).isEqualTo("8.4");
+		assertThat(doc.getFieldValue("ratings_age")).isEqualTo("TV-14");
+		assertThat(doc.getFieldValue("seasons")).isEqualTo("2");
+		assertThat(doc.getFieldValue("ongoing")).isEqualTo("false");
+		assertThat(doc.getFieldValue("released")).isEqualTo("2022-09-21");
+		assertThat(doc.getFieldNames()).doesNotContain("empty");
+		assertThat(doc.getFieldValue("content")).asString().contains("Prequel").doesNotContain("show-1");
+	}
+
+	@Test
+	void invalidYamlIsReported() {
+		assertThatThrownBy(() -> creator.create("---\ntitle: [unclosed\n---\nbody\n"))
+				.isInstanceOf(DocumentProcessingException.class).hasMessageContaining("YAML");
+	}
+
+	@Test
+	void documentWithoutFrontMatterIsUnchanged() {
+		SolrInputDocument doc = creator.create("# Just a heading\n\nBody text.\n").getFirst();
+
+		assertThat(doc.getFieldValue("title")).isEqualTo("Just a heading");
+		assertThat(doc.getFieldValue("content")).asString().contains("Body text");
+		assertThat(doc.getFieldValue("id")).isNotNull();
+	}
+}