diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cbbb98..aadcd73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Add a pluggable post-download `ContentProcessor` registry and built-in + `PdfProcessor` that extracts Markdown from text-based PDFs with local + `pdf-inspector` processing and explicit OCR guidance for unsupported pages. - Discover agent-facing resources from HTTP and HTML declarations plus bounded conventional probes, including `llms.txt`, `auth.md`, OAuth metadata, MCP server cards, A2A agent cards, and Agent Skills indexes. Markdown responses diff --git a/Cargo.lock b/Cargo.lock index 1296c06..1944d41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -177,6 +188,12 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.13.0" @@ -201,6 +218,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "brotli" version = "8.0.4" @@ -240,6 +266,15 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.2.67" @@ -288,6 +323,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "clap" version = "4.6.1" @@ -445,6 +490,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + [[package]] name = "crypto-common" version = "0.1.7" @@ -532,6 +602,37 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2 1.0.106", + "quote 1.0.46", + "syn 2.0.118", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror", +] + [[package]] name = "der" version = "0.7.10" @@ -542,6 +643,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + [[package]] name = "derive_more" version = "2.1.1" @@ -645,6 +752,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ecb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7" +dependencies = [ + "cipher", +] + [[package]] name = "ed25519" version = "2.2.3" @@ -691,6 +807,29 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -724,6 +863,7 @@ dependencies = [ "futures", "libc", "mira-eval", + "pdf-inspector", "percent-encoding", "rakers", "rand 0.10.2", @@ -1262,6 +1402,25 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2 1.0.106", + "quote 1.0.46", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -1272,6 +1431,16 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "inventory" version = "0.3.24" @@ -1299,6 +1468,59 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2 1.0.106", + "quote 1.0.46", + "syn 2.0.118", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jni" version = "0.22.4" @@ -1408,6 +1630,37 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "lopdf" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67513274c50a2b51e5f75d9e682fcf4ab064a8a9c9ae2c3c59309084882bb24d" +dependencies = [ + "aes", + "bitflags 2.13.0", + "cbc", + "chrono", + "ecb", + "encoding_rs", + "flate2", + "getrandom 0.4.3", + "indexmap", + "itoa", + "jiff", + "log", + "md-5", + "nom", + "rand 0.10.2", + "rangemap", + "rayon", + "sha2 0.10.9", + "stringprep", + "thiserror", + "time", + "ttf-parser", + "weezl", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -1466,6 +1719,16 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + [[package]] name = "memchr" version = "2.8.3" @@ -1532,6 +1795,15 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1541,6 +1813,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-traits" version = "0.2.19" @@ -1601,6 +1879,24 @@ dependencies = [ "windows-link", ] +[[package]] +name = "pdf-inspector" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7475018de0880b394b7cc50f871fac0c010aa411dcd14c64074a3e640a4c05c" +dependencies = [ + "env_logger", + "include_dir", + "log", + "lopdf", + "once_cell", + "rayon", + "regex", + "thiserror", + "ttf-parser", + "unicode-normalization", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1726,6 +2022,15 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -1735,6 +2040,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "precomputed-hash" version = "0.1.1" @@ -1968,13 +2279,39 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rangemap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.13.0", ] [[package]] @@ -2165,7 +2502,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.13.0", "errno", "libc", "linux-raw-sys", @@ -2325,7 +2662,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags", + "bitflags 2.13.0", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -2348,7 +2685,7 @@ version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8adfa1c298912827b8a28b223b3b874357397ae706e6190acd9bf28cee99114d" dependencies = [ - "bitflags", + "bitflags 2.13.0", "cssparser", "derive_more", "log", @@ -2618,6 +2955,17 @@ dependencies = [ "quote 1.0.46", ] +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -2678,7 +3026,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags", + "bitflags 2.13.0", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -2761,6 +3109,36 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -2897,7 +3275,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", - "bitflags", + "bitflags 2.13.0", "bytes", "futures-core", "futures-util", @@ -2992,18 +3370,45 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -3239,6 +3644,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 9bff10e..581c34e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,9 @@ async-trait = "0.1" # Service trait tower = { version = "0.5", features = ["util"] } +# Binary document extraction +pdf-inspector = "0.1.7" + # Platform filesystem flags libc = "0.2" diff --git a/README.md b/README.md index 2de08f9..877942d 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,8 @@ AI-friendly web content fetching tool designed for LLM consumption. Rust library - **Agent content focus** - Optional low-noise extraction mode for AI agents - **Crawl discovery** - Optional bounded same-origin page discovery for AI agents - **HTML-to-Text** - Plain text extraction with clean formatting -- **Binary detection** - Returns metadata only for images, PDFs, etc. +- **Content processors** - Post-download extraction for text PDFs, with an extensible registry +- **Binary detection** - Returns metadata only for unsupported binary formats - **Timeout handling** - 1s first-byte, 30s body with partial content on timeout - **Safety limits** - 10 MB default decompressed body cap with truncation - **URL filtering** - URL-aware allow/block lists for controlled access @@ -36,6 +37,21 @@ fetchers match first; the default fetcher handles everything else. - `DocsSiteFetcher` - docs sites with `llms.txt`/`llms-full.txt` support - `DefaultFetcher` - all remaining HTTP/HTTPS URLs with HTML conversion, streaming, timeout handling, and binary detection +## Built-in Content Processors + +Content processors run after a fetcher retrieves a bounded response body. They +select by final URL and response media type, then turn non-text formats into +LLM-friendly content without performing their own network requests. + +- `PdfProcessor` - text-based PDF classification and Markdown extraction via + [`pdf-inspector`](https://github.com/firecrawl/pdf-inspector); scanned or + image-only pages are reported as requiring OCR + +Custom processors implement `ContentProcessor` and can be registered in a +`ContentProcessorRegistry`. Pass that registry to +`FetcherRegistry::with_content_processors` to retain the built-in fetchers while +customizing post-download processing. + ## Installation ### From crates.io (recommended) @@ -278,7 +294,7 @@ Errors are returned in the `error` field: - `Timeout` - Request timed out - `HttpError` - 4xx/5xx response - `ContentError` - Failed to read body -- `BinaryContent` - Binary content not supported +- `BinaryContent` - Binary content has no registered processor ## Security @@ -313,11 +329,16 @@ See [`docs/hardening.md`](docs/hardening.md) for deployment guidance. Partial content is returned on body timeout or body-size limit with `truncated: true`. -### Binary Content +### PDF And Binary Content + +When Markdown is requested, text-based PDFs are downloaded within the configured +body limit and converted to Markdown. PDF parsing runs locally; no OCR service or +additional network request is used. Scanned/image-only PDFs report `use_ocr` as the +suggested next action. -Automatically detected and returns metadata only for: +Other binary content returns metadata only: - Images, audio, video, fonts -- PDFs, archives (zip, tar, rar, 7z) +- Archives (zip, tar, rar, 7z) - Office documents ### HTML Conversion diff --git a/crates/fetchkit/Cargo.toml b/crates/fetchkit/Cargo.toml index acdf886..9b231e6 100644 --- a/crates/fetchkit/Cargo.toml +++ b/crates/fetchkit/Cargo.toml @@ -30,6 +30,7 @@ futures = { workspace = true } bytes = { workspace = true } async-trait = { workspace = true } tower = { workspace = true } +pdf-inspector = { workspace = true } libc = { workspace = true } rakers = { workspace = true, optional = true } diff --git a/crates/fetchkit/src/content.rs b/crates/fetchkit/src/content.rs new file mode 100644 index 0000000..2843f22 --- /dev/null +++ b/crates/fetchkit/src/content.rs @@ -0,0 +1,287 @@ +//! Post-download content processing. +//! +//! Fetchers own retrieval and network policy. Content processors operate only on +//! bounded response bytes, so format-specific extraction cannot bypass egress controls. + +use async_trait::async_trait; +use bytes::Bytes; +use pdf_inspector::process_pdf_mem; +use thiserror::Error; +use tokio::sync::Semaphore; +use url::Url; + +use crate::{PageMetadata, PageQuality}; + +// pdf-inspector uses Rayon internally; bound concurrent documents so batch fetches +// cannot multiply parser thread pools without limit. +static PDF_PROCESSING_LIMIT: Semaphore = Semaphore::const_new(2); + +/// Bounded response bytes and metadata passed to a [`ContentProcessor`]. +pub struct ContentProcessorInput { + /// Final URL after redirects. + pub url: Url, + /// Response Content-Type header, when present. + pub content_type: Option, + /// Response body after fetchkit's timeout and size limits were applied. + pub body: Bytes, +} + +/// Content extracted from a non-text response. +#[derive(Debug, Default)] +pub struct ProcessedContent { + /// Output format, such as `markdown`. + pub format: Option, + /// Extracted textual content. + pub content: Option, + /// Structured metadata discovered while processing. + pub metadata: Option, + /// Agent-facing quality signals. + pub quality: Option, + /// Recoverable processing limitation, such as OCR being required. + pub error: Option, +} + +/// Error returned when a content processor cannot parse its input. +#[derive(Debug, Error)] +#[error("{0}")] +pub struct ContentProcessorError(pub String); + +/// Converts bounded response bytes into LLM-friendly textual content. +#[async_trait] +pub trait ContentProcessor: Send + Sync { + /// Unique identifier for logging and diagnostics. + fn name(&self) -> &'static str; + + /// Whether this processor supports the response metadata. + /// + /// This is evaluated before downloading a body that would otherwise be + /// rejected as binary, so implementations must not inspect body bytes here. + fn matches(&self, url: &Url, content_type: Option<&str>) -> bool; + + /// Process an already-downloaded, bounded response body. + async fn process( + &self, + input: ContentProcessorInput, + ) -> Result; +} + +/// Ordered registry of response content processors. +pub struct ContentProcessorRegistry { + processors: Vec>, +} + +impl Default for ContentProcessorRegistry { + fn default() -> Self { + Self::new() + } +} + +impl ContentProcessorRegistry { + /// Create an empty registry. + pub fn new() -> Self { + Self { + processors: Vec::new(), + } + } + + /// Create a registry containing fetchkit's built-in processors. + pub fn with_defaults() -> Self { + let mut registry = Self::new(); + registry.register(Box::new(PdfProcessor)); + registry + } + + /// Append a processor. The first matching processor wins. + pub fn register(&mut self, processor: Box) { + self.processors.push(processor); + } + + /// Find the first processor matching response metadata. + pub fn find(&self, url: &Url, content_type: Option<&str>) -> Option<&dyn ContentProcessor> { + self.processors + .iter() + .find(|processor| processor.matches(url, content_type)) + .map(Box::as_ref) + } +} + +/// Extracts native text PDFs as Markdown using `pdf-inspector`. +pub struct PdfProcessor; + +#[async_trait] +impl ContentProcessor for PdfProcessor { + fn name(&self) -> &'static str { + "pdf" + } + + fn matches(&self, url: &Url, content_type: Option<&str>) -> bool { + let media_type = content_type + .and_then(|value| value.split(';').next()) + .map(str::trim) + .unwrap_or_default(); + if media_type.eq_ignore_ascii_case("application/pdf") + || media_type.eq_ignore_ascii_case("application/x-pdf") + { + return true; + } + + (media_type.is_empty() || media_type.eq_ignore_ascii_case("application/octet-stream")) + && url.path().to_ascii_lowercase().ends_with(".pdf") + } + + async fn process( + &self, + input: ContentProcessorInput, + ) -> Result { + let _permit = PDF_PROCESSING_LIMIT.acquire().await.map_err(|error| { + ContentProcessorError(format!("PDF processing unavailable: {error}")) + })?; + let result = tokio::task::spawn_blocking(move || process_pdf_mem(&input.body)) + .await + .map_err(|error| ContentProcessorError(format!("PDF processor task failed: {error}")))? + .map_err(|error| ContentProcessorError(error.to_string()))?; + + let needs_ocr = !result.pages_needing_ocr.is_empty(); + let mut warnings = Vec::new(); + if needs_ocr { + warnings.push("pdf_requires_ocr".to_string()); + } + if result.has_encoding_issues { + warnings.push("pdf_encoding_issues".to_string()); + } + + let content = result.markdown.filter(|value| !value.trim().is_empty()); + let error = content.is_none().then(|| { + if needs_ocr { + format!( + "PDF requires OCR for pages: {}", + comma_separated_pages(&result.pages_needing_ocr) + ) + } else { + "PDF contains no extractable text".to_string() + } + }); + let metadata = PageMetadata { + title: result.title, + extraction_method: Some("pdf_inspector".to_string()), + ..Default::default() + }; + let quality = PageQuality { + score: if content.is_none() { + 0.0 + } else if warnings.is_empty() { + 1.0 + } else { + 0.6 + }, + warnings, + extraction_method: Some("pdf_inspector".to_string()), + suggested_next_action: needs_ocr.then(|| "use_ocr".to_string()), + ..Default::default() + }; + + Ok(ProcessedContent { + format: content.as_ref().map(|_| "markdown".to_string()), + content, + metadata: Some(metadata), + quality: Some(quality), + error, + }) + } +} + +fn comma_separated_pages(pages: &[u32]) -> String { + pages + .iter() + .map(u32::to_string) + .collect::>() + .join(", ") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn text_pdf(text: &str) -> Bytes { + let escaped = text + .replace('\\', "\\\\") + .replace('(', "\\(") + .replace(')', "\\)"); + let stream = format!("BT /F1 24 Tf 72 720 Td ({escaped}) Tj ET"); + let objects = [ + "<< /Type /Catalog /Pages 2 0 R >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>".to_string(), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(), + format!("<< /Length {} >>\nstream\n{stream}\nendstream", stream.len()), + ]; + + let mut pdf = b"%PDF-1.4\n".to_vec(); + let mut offsets = Vec::new(); + for (index, object) in objects.iter().enumerate() { + offsets.push(pdf.len()); + pdf.extend_from_slice(format!("{} 0 obj\n{object}\nendobj\n", index + 1).as_bytes()); + } + let xref_offset = pdf.len(); + pdf.extend_from_slice(format!("xref\n0 {}\n", objects.len() + 1).as_bytes()); + pdf.extend_from_slice(b"0000000000 65535 f \n"); + for offset in offsets { + pdf.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes()); + } + pdf.extend_from_slice( + format!( + "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref_offset}\n%%EOF\n", + objects.len() + 1 + ) + .as_bytes(), + ); + Bytes::from(pdf) + } + + #[test] + fn pdf_processor_matches_media_type_or_binary_pdf_path() { + let processor = PdfProcessor; + let extensionless = Url::parse("https://example.com/download?id=1").unwrap(); + let pdf_path = Url::parse("https://example.com/report.PDF").unwrap(); + + assert!(processor.matches(&extensionless, Some("application/pdf; charset=binary"))); + assert!(processor.matches(&pdf_path, Some("application/octet-stream"))); + assert!(!processor.matches(&pdf_path, Some("text/html"))); + assert!(!processor.matches(&extensionless, Some("application/octet-stream"))); + } + + #[test] + fn registry_uses_first_matching_processor() { + let registry = ContentProcessorRegistry::with_defaults(); + let url = Url::parse("https://example.com/document").unwrap(); + + assert_eq!( + registry + .find(&url, Some("application/pdf")) + .map(ContentProcessor::name), + Some("pdf") + ); + assert!(registry.find(&url, Some("image/png")).is_none()); + } + + #[tokio::test] + async fn pdf_processor_extracts_markdown_from_memory() { + let processor = PdfProcessor; + let result = processor + .process(ContentProcessorInput { + url: Url::parse("https://example.com/document").unwrap(), + content_type: Some("application/pdf".to_string()), + body: text_pdf("Hello from PDF"), + }) + .await + .unwrap(); + + assert_eq!(result.format.as_deref(), Some("markdown")); + assert!(result.content.unwrap().contains("Hello from PDF")); + assert_eq!( + result.metadata.unwrap().extraction_method.as_deref(), + Some("pdf_inspector") + ); + assert!(result.error.is_none()); + } +} diff --git a/crates/fetchkit/src/fetchers/default.rs b/crates/fetchkit/src/fetchers/default.rs index 9d7f7fe..4aa6e88 100644 --- a/crates/fetchkit/src/fetchers/default.rs +++ b/crates/fetchkit/src/fetchers/default.rs @@ -8,6 +8,7 @@ //! specialized fetchers. use crate::client::FetchOptions; +use crate::content::{ContentProcessorInput, ContentProcessorRegistry}; use crate::convert::{ classify_agent_resource, extract_headings, extract_metadata, extract_readable_content, filter_excessive_newlines, html_to_markdown_with_base_url, html_to_text, is_html, @@ -72,6 +73,7 @@ const BINARY_PREFIXES: &[&str] = &[ "video/", "application/octet-stream", "application/pdf", + "application/x-pdf", "application/zip", "application/gzip", "application/x-tar", @@ -126,12 +128,19 @@ pub(crate) const DEFAULT_MAX_BODY_SIZE: usize = 10 * 1024 * 1024; /// - HTML to markdown/text conversion /// - Binary content detection /// - Timeout handling with partial content -pub struct DefaultFetcher; +pub struct DefaultFetcher { + content_processors: ContentProcessorRegistry, +} impl DefaultFetcher { /// Create a new default fetcher pub fn new() -> Self { - Self + Self::with_content_processors(ContentProcessorRegistry::with_defaults()) + } + + /// Create a default fetcher with a custom content processor registry. + pub fn with_content_processors(content_processors: ContentProcessorRegistry) -> Self { + Self { content_processors } } } @@ -508,7 +517,107 @@ impl Fetcher for DefaultFetcher { }); } - // Check for binary content + let parsed_final_url = Url::parse(&final_url).map_err(|_| FetchError::InvalidUrlScheme)?; + let content_processor = wants_markdown + .then(|| { + self.content_processors + .find(&parsed_final_url, meta.content_type.as_deref()) + }) + .flatten(); + + // Content processors receive bytes only after the network path applies its + // timeout and decompressed-size limits. + if let Some(processor) = content_processor { + let processor_name = processor.name(); + let (body, input_truncated) = + read_body_with_timeout(response, BODY_TIMEOUT, max_body_size).await?; + let size = body.len() as u64; + + if input_truncated { + let mut quality = binary_quality_signal(); + quality.warnings.push("truncated".to_string()); + return Ok(FetchResponse { + url: final_url, + status_code, + content_type: meta.content_type, + size: Some(size), + last_modified: meta.last_modified, + etag: meta.etag, + filename: meta.filename, + truncated: Some(true), + redirect_chain, + error: Some(format!( + "Content processor {processor_name} cannot process a partial response body" + )), + quality: Some(quality), + ..Default::default() + }); + } + + debug!(processor = processor_name, "Processing response content"); + let processed = processor + .process(ContentProcessorInput { + url: parsed_final_url, + content_type: meta.content_type.clone(), + body, + }) + .await; + + return match processed { + Ok(mut processed) => { + let output_truncated = processed + .content + .as_mut() + .map(|content| truncate_string_to_max_bytes(content, max_body_size)) + .unwrap_or(false); + if output_truncated { + if let Some(content) = &mut processed.content { + content.push_str(TRUNCATION_MESSAGE); + } + if let Some(quality) = &mut processed.quality { + push_warning(&mut quality.warnings, "truncated"); + quality.score = (quality.score - 0.2).max(0.0); + } + } + let word_count = processed.content.as_deref().map(count_words); + Ok(FetchResponse { + url: final_url, + status_code, + content_type: meta.content_type, + size: Some(size), + last_modified: meta.last_modified, + etag: meta.etag, + filename: meta.filename, + format: processed.format, + content: processed.content, + truncated: output_truncated.then_some(true), + metadata: processed.metadata, + quality: processed.quality, + word_count, + redirect_chain, + error: processed.error, + ..Default::default() + }) + } + Err(error) => Ok(FetchResponse { + url: final_url, + status_code, + content_type: meta.content_type, + size: Some(size), + last_modified: meta.last_modified, + etag: meta.etag, + filename: meta.filename, + redirect_chain, + error: Some(format!( + "Content processor {processor_name} failed: {error}" + )), + quality: Some(binary_quality_signal()), + ..Default::default() + }), + }; + } + + // Check for unsupported binary content if let Some(ref ct) = meta.content_type { if is_binary_content_type(ct) { return Ok(FetchResponse { @@ -1354,6 +1463,7 @@ mod tests { assert!(is_binary_content_type("audio/mp3")); assert!(is_binary_content_type("video/mp4")); assert!(is_binary_content_type("application/pdf")); + assert!(is_binary_content_type("application/x-pdf")); assert!(is_binary_content_type("application/octet-stream")); assert!(is_binary_content_type("application/zip")); assert!(is_binary_content_type("application/vnd.ms-excel")); diff --git a/crates/fetchkit/src/fetchers/mod.rs b/crates/fetchkit/src/fetchers/mod.rs index f506c25..76cabf0 100644 --- a/crates/fetchkit/src/fetchers/mod.rs +++ b/crates/fetchkit/src/fetchers/mod.rs @@ -48,6 +48,7 @@ pub use wikipedia::WikipediaFetcher; pub use youtube::YouTubeFetcher; use crate::client::FetchOptions; +use crate::content::ContentProcessorRegistry; use crate::error::FetchError; use crate::file_saver::FileSaver; use crate::types::{FetchRequest, FetchResponse}; @@ -159,6 +160,11 @@ impl FetcherRegistry { /// 9. DocsSiteFetcher - handles docs sites and llms.txt URLs /// 10. DefaultFetcher - handles all remaining HTTP/HTTPS URLs pub fn with_defaults() -> Self { + Self::with_content_processors(ContentProcessorRegistry::with_defaults()) + } + + /// Create the default fetcher registry with a custom content processor registry. + pub fn with_content_processors(content_processors: ContentProcessorRegistry) -> Self { let mut registry = Self::new(); // Register specialized fetchers first (higher priority) // Notebooks precede the generic GitHub and GitLab blob fetchers. @@ -185,7 +191,9 @@ impl FetcherRegistry { // DocsSiteFetcher for docs sites and llms.txt registry.register(Box::new(DocsSiteFetcher::new())); // Default fetcher last (catches all remaining URLs) - registry.register(Box::new(DefaultFetcher::new())); + registry.register(Box::new(DefaultFetcher::with_content_processors( + content_processors, + ))); registry } diff --git a/crates/fetchkit/src/lib.rs b/crates/fetchkit/src/lib.rs index efc1697..ab9c0e1 100644 --- a/crates/fetchkit/src/lib.rs +++ b/crates/fetchkit/src/lib.rs @@ -56,6 +56,8 @@ //! Fetchkit uses a pluggable fetcher system where specialized fetchers //! handle specific URL patterns. The [`FetcherRegistry`] dispatches //! requests to the appropriate fetcher based on URL matching. +//! After retrieval, the [`ContentProcessorRegistry`] can transform bounded +//! non-text response bytes based on media type. //! //! Built-in fetchers: //! - [`ArXivFetcher`] - arXiv paper metadata and abstract @@ -76,6 +78,7 @@ pub mod bot_auth; pub mod client; +pub mod content; mod convert; mod crawl; mod dns; @@ -87,6 +90,10 @@ pub mod transport; mod types; pub use client::{batch_fetch, batch_fetch_with_options, fetch, fetch_with_options, FetchOptions}; +pub use content::{ + ContentProcessor, ContentProcessorError, ContentProcessorInput, ContentProcessorRegistry, + PdfProcessor, ProcessedContent, +}; pub use convert::{ extract_headings, extract_metadata, extract_readable_content, html_to_markdown, html_to_markdown_with_base_url, html_to_text, strip_boilerplate, @@ -120,7 +127,7 @@ pub const DEFAULT_USER_AGENT: &str = "Everruns Fetchkit/1.0"; /// Backward-compatible full description string with file-saving enabled. pub const TOOL_DESCRIPTION: &str = - "Fetch URL content as text or markdown; return metadata for binary responses or save bytes to file."; + "Fetch URL content as text or markdown, including text-based PDFs; return metadata for unsupported binary responses or save bytes to file."; /// Backward-compatible help document with file-saving enabled. pub static TOOL_LLMTXT: std::sync::LazyLock = diff --git a/crates/fetchkit/src/tool.rs b/crates/fetchkit/src/tool.rs index bd85076..ed94984 100644 --- a/crates/fetchkit/src/tool.rs +++ b/crates/fetchkit/src/tool.rs @@ -969,24 +969,24 @@ fn display_name(locale: &str) -> &'static str { fn description(locale: &str, enable_save_to_file: bool) -> String { if is_ukrainian(locale) { if enable_save_to_file { - "Завантажити URL як текст або markdown; повернути метадані для бінарного вмісту або зберегти байти через save_to_file.".to_string() + "Завантажити URL як текст або markdown, включно з текстовими PDF; повернути метадані для непідтримуваного бінарного вмісту або зберегти байти через save_to_file.".to_string() } else { - "Завантажити URL як текст або markdown; повернути метадані для бінарного вмісту." + "Завантажити URL як текст або markdown, включно з текстовими PDF; повернути метадані для непідтримуваного бінарного вмісту." .to_string() } } else if enable_save_to_file { - "Fetch URL content as text or markdown; return metadata for binary responses or save bytes with save_to_file.".to_string() + "Fetch URL content as text or markdown, including text-based PDFs; return metadata for unsupported binary responses or save bytes with save_to_file.".to_string() } else { - "Fetch URL content as text or markdown; return metadata for binary responses.".to_string() + "Fetch URL content as text or markdown, including text-based PDFs; return metadata for unsupported binary responses.".to_string() } } fn system_prompt(locale: &str, enable_save_to_file: bool, block_private_ips: bool) -> String { if is_ukrainian(locale) { let binary_rule = if enable_save_to_file { - "Бінарні відповіді повертають метадані; використовуйте save_to_file, щоб зберегти байти." + "Непідтримувані бінарні відповіді повертають метадані; використовуйте save_to_file, щоб зберегти байти." } else { - "Бінарні відповіді повертають лише метадані." + "Непідтримувані бінарні відповіді повертають лише метадані." }; let network_rule = if block_private_ips { "Приватні IP-адреси заблоковані." @@ -999,9 +999,9 @@ fn system_prompt(locale: &str, enable_save_to_file: bool, block_private_ips: boo ) } else { let binary_rule = if enable_save_to_file { - "Binary responses return metadata; use save_to_file to persist bytes." + "Unsupported binary responses return metadata; use save_to_file to persist bytes." } else { - "Binary responses return metadata only." + "Unsupported binary responses return metadata only." }; let network_rule = if block_private_ips { "Private IPs are blocked." diff --git a/crates/fetchkit/src/types.rs b/crates/fetchkit/src/types.rs index c5a012a..5b2bef7 100644 --- a/crates/fetchkit/src/types.rs +++ b/crates/fetchkit/src/types.rs @@ -346,10 +346,9 @@ pub struct AgentResource { pub verified: bool, } -/// Structured metadata extracted from an HTML page. +/// Structured metadata extracted from page or document content. /// -/// All fields are optional — only populated when the corresponding -/// HTML elements or meta tags are present. +/// All fields are optional and populated only when available for the source type. #[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)] pub struct PageMetadata { /// Page title from `` or `og:title` @@ -571,7 +570,7 @@ pub struct FetchResponse { #[serde(skip_serializing_if = "Option::is_none")] pub bytes_written: Option<u64>, - /// Structured page metadata extracted from HTML + /// Structured page or document metadata #[serde(skip_serializing_if = "Option::is_none")] pub metadata: Option<PageMetadata>, diff --git a/crates/fetchkit/tests/integration.rs b/crates/fetchkit/tests/integration.rs index 11b8c58..05d23c5 100644 --- a/crates/fetchkit/tests/integration.rs +++ b/crates/fetchkit/tests/integration.rs @@ -1,8 +1,9 @@ //! Integration tests for Fetchkit using wiremock use fetchkit::{ - fetch_with_options, DnsPolicy, FetchError, FetchOptions, FetchRequest, FetcherRegistry, - FileSaveError, FileSaver, HttpMethod, LocalFileSaver, SaveResult, Tool, + fetch_with_options, ContentProcessorRegistry, DnsPolicy, FetchError, FetchOptions, + FetchRequest, FetcherRegistry, FileSaveError, FileSaver, HttpMethod, LocalFileSaver, + SaveResult, Tool, }; use serde_json::json; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -34,6 +35,42 @@ fn test_tool_with_save() -> Tool { .build() } +fn text_pdf(text: &str) -> Vec<u8> { + let escaped = text + .replace('\\', "\\\\") + .replace('(', "\\(") + .replace(')', "\\)"); + let stream = format!("BT /F1 24 Tf 72 720 Td ({escaped}) Tj ET"); + let objects = [ + "<< /Type /Catalog /Pages 2 0 R >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>".to_string(), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(), + format!("<< /Length {} >>\nstream\n{stream}\nendstream", stream.len()), + ]; + + let mut pdf = b"%PDF-1.4\n".to_vec(); + let mut offsets = Vec::new(); + for (index, object) in objects.iter().enumerate() { + offsets.push(pdf.len()); + pdf.extend_from_slice(format!("{} 0 obj\n{object}\nendobj\n", index + 1).as_bytes()); + } + let xref_offset = pdf.len(); + pdf.extend_from_slice(format!("xref\n0 {}\n", objects.len() + 1).as_bytes()); + pdf.extend_from_slice(b"0000000000 65535 f \n"); + for offset in offsets { + pdf.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes()); + } + pdf.extend_from_slice( + format!( + "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref_offset}\n%%EOF\n", + objects.len() + 1 + ) + .as_bytes(), + ); + pdf +} + struct RejectingFileSaver; #[async_trait::async_trait] @@ -399,6 +436,57 @@ async fn test_binary_content() { ); } +#[tokio::test] +async fn test_pdf_content_processor_returns_markdown() { + let mock_server = MockServer::start().await; + let body = text_pdf("Hello from fetched PDF"); + + Mock::given(method("GET")) + .and(path("/download")) + .respond_with( + ResponseTemplate::new(200) + .set_body_bytes(body.clone()) + .insert_header("content-type", "application/pdf"), + ) + .mount(&mock_server) + .await; + + let req = FetchRequest::new(format!("{}/download", mock_server.uri())).as_markdown(); + let resp = fetch_with_options(req, test_options()).await.unwrap(); + + assert_eq!(resp.content_type.as_deref(), Some("application/pdf")); + assert_eq!(resp.size, Some(body.len() as u64)); + assert_eq!(resp.format.as_deref(), Some("markdown")); + assert!(resp.content.unwrap().contains("Hello from fetched PDF")); + assert_eq!( + resp.metadata.unwrap().extraction_method.as_deref(), + Some("pdf_inspector") + ); + assert!(resp.error.is_none()); +} + +#[tokio::test] +async fn test_custom_content_processor_registry_can_disable_pdf_processing() { + let mock_server = MockServer::start().await; + + Mock::given(method("GET")) + .and(path("/document.pdf")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "application/pdf") + .insert_header("content-length", "4"), + ) + .mount(&mock_server) + .await; + + let registry = FetcherRegistry::with_content_processors(ContentProcessorRegistry::new()); + let req = FetchRequest::new(format!("{}/document.pdf", mock_server.uri())).as_markdown(); + let resp = registry.fetch(req, test_options()).await.unwrap(); + + assert!(resp.content.is_none()); + assert!(resp.error.unwrap().contains("Binary content")); +} + #[tokio::test] async fn test_4xx_status() { let mock_server = MockServer::start().await; diff --git a/specs/fetchers.md b/specs/fetchers.md index 971cd87..f4fce99 100644 --- a/specs/fetchers.md +++ b/specs/fetchers.md @@ -2,7 +2,10 @@ ## Abstract -Fetcher system enables specialized content fetching based on URL patterns. Each fetcher handles specific URL types (e.g., GitHub repos, binary files) with custom logic, returning structured responses optimized for LLM consumption. +Fetcher system enables specialized content retrieval based on URL patterns. A sibling content +processor system transforms already-retrieved response bytes based on media type. This keeps +network policy in fetchers while allowing formats such as PDF to produce structured responses +optimized for LLM consumption. ## Requirements @@ -27,6 +30,34 @@ Central dispatcher that: 6. Provides shared URL-policy validation for fetchers that derive secondary API URLs; every outbound destination must satisfy the same policy before transport execution 7. Provides `fetch_to_file()` that dispatches to matched fetcher's `fetch_to_file()` +### Content Processor Trait And Registry + +Each content processor must implement: + +1. **`name()`** - Unique identifier for logging/debugging. +2. **`matches(url, content_type)`** - Returns true when response metadata identifies + a supported format. Matching occurs before a body that would otherwise be rejected + as binary is downloaded. +3. **`process(input)`** - Async processing of final URL, Content-Type, and body bytes. + Input bytes have already passed fetchkit's timeout and decompressed-size limits. + +`ContentProcessorRegistry` is ordered and uses the first matching processor. It provides +`new()`, `with_defaults()`, `register()`, and `find()`. Processors do not perform network +requests. `FetcherRegistry::with_content_processors()` retains the built-in fetchers while +allowing callers to replace the default content processor registry. + +#### PdfProcessor + +- Matches PDF response media types, including extensionless download URLs. A `.pdf` + suffix is a fallback only for missing Content-Type or `application/octet-stream`. +- Uses `pdf-inspector` to classify and extract Markdown from bounded in-memory bytes. +- Runs CPU work through `spawn_blocking`, with at most two PDF documents processed + concurrently per process. +- Does not perform OCR. Responses identify OCR-required or encoding-problem pages + through quality warnings and `suggested_next_action: "use_ocr"`. +- Extracted output is capped by the same configured `max_body_size`; partial PDF input + is not passed to the parser. + ### Built-in Fetchers #### DefaultFetcher (lowest priority) @@ -36,7 +67,8 @@ Central dispatcher that: - Features: - GET and HEAD methods - HTML to markdown/text conversion (when enabled) - - Binary content detection (returns metadata only) + - Content processor dispatch before unsupported binary detection + - Binary content detection (returns metadata only when no processor matches) - Timeout handling with partial content support - Binary-aware file saving via `fetch_to_file()` override (accepts binary content when saving) - Decompressed body size cap with partial content truncation @@ -328,6 +360,7 @@ Both built-in fetchers integrate resolve-then-check DNS validation: ``` crates/fetchkit/src/ +├── content.rs # ContentProcessor trait, registry, PDF processor ├── dns.rs # DnsPolicy - SSRF prevention via resolve-then-check ├── file_saver.rs # FileSaver trait, LocalFileSaver, SaveResult, FileSaveError ├── fetchers/ diff --git a/specs/initial.md b/specs/initial.md index f70ba67..6136f40 100644 --- a/specs/initial.md +++ b/specs/initial.md @@ -9,9 +9,9 @@ ## Abstract Define a standalone Rust crate named `fetchkit` that implements the existing Fetchkit tool -behavior: fetch URL content, optional HTML conversion, strict timeouts, and metadata-only -responses for binary content. The crate also ships a CLI, an MCP server, and Python bindings -that expose the same tool contract. +behavior: fetch URL content, optional HTML and document conversion, strict timeouts, and +metadata-only responses for unsupported binary content. The crate also ships a CLI, an MCP +server, and Python bindings that expose the same tool contract. ## Requirements @@ -206,7 +206,7 @@ Provide a builder to configure tool options, including: - Expose a single `web_fetch` tool over MCP. - Input schema: derived from `FetchRequest` via tool builder (disabled options omitted). - Output: Markdown with YAML frontmatter (same format as CLI `--output md`). -- Tool description: "Fetch URL content as text or markdown; return metadata for binary responses or save bytes to file." +- Tool description: "Fetch URL content as text or markdown, including text-based PDFs; return metadata for unsupported binary responses or save bytes to file." ### Python Bindings @@ -304,10 +304,21 @@ By default, Fetchkit blocks connections to private/reserved IP ranges: #### Binary Content - Detect binary by Content-Type prefix: - - `image/`, `audio/`, `video/`, `application/octet-stream`, `application/pdf`, + - `image/`, `audio/`, `video/`, `application/octet-stream`, `application/pdf`, `application/x-pdf`, `application/zip`, `application/gzip`, `application/x-tar`, `application/x-rar`, `application/x-7z`, `application/vnd.ms-`, `application/vnd.openxmlformats`, `font/`. -- For binary: +- When `as_markdown` is requested, a matching registered content processor runs + on the bounded response body before the generic binary response is returned. +- The built-in PDF processor: + - Matches `application/pdf`, `application/x-pdf`, or a `.pdf` URL served as + `application/octet-stream`/without Content-Type. + - Uses `pdf-inspector` on in-memory bytes and returns `format: "markdown"` for + PDFs with an extractable text layer. + - Performs no network requests and no OCR. Scanned/image-only pages produce + `pdf_requires_ocr` quality warnings and suggest `use_ocr`. + - Runs on a bounded blocking pool path, and both PDF input and extracted output + honor `max_body_size`. Partial PDF inputs are not parsed. +- For binary without a matching processor: - Return metadata (`content_type`, `size`, `filename`, `last_modified`) - Include `error: "Binary content is not supported. Only textual content (HTML, text, JSON, etc.) can be fetched."` - Omit `content`, `format`, `truncated` @@ -420,7 +431,7 @@ Content is HTML if: Unit: - URL validation, method parsing. -- Binary content detection. +- Binary content detection and content-processor dispatch. - HTML conversion, entity decoding. - Newline filtering behavior. - DNS policy IP range blocking (IPv4, IPv6, mapped addresses). @@ -428,7 +439,7 @@ Unit: Integration (mock HTTP server): - GET/HEAD with expected fields. - HTML -> markdown/text conversion. -- Binary content metadata response. +- PDF-to-Markdown response and unsupported binary metadata response. - 4xx/5xx status handling. - Last-Modified extraction. - Size correctness for text and binary. diff --git a/specs/threat-model.md b/specs/threat-model.md index e4fcf64..dc07c90 100644 --- a/specs/threat-model.md +++ b/specs/threat-model.md @@ -287,6 +287,7 @@ to protect against changes between validation and write. | TM-DOS-004 | Rapid request flooding via tool | Low | No rate limiting in Fetchkit; caller responsibility | **CALLER RISK** | | TM-DOS-005 | DNS resolution delay | Low | DNS resolution uses system resolver; no explicit timeout on DNS lookup | **ACCEPTED** | | TM-DOS-006 | Memory exhaustion from large HTML conversion | Medium | Conversion input bounded by `max_body_size` (10 MB default) | MITIGATED | +| TM-DOS-007 | CPU/memory exhaustion from adversarial PDF parsing | Medium | Bounded input/output, blocking-task isolation, and two-document concurrency cap; parser-internal expansion remains possible | **ACCEPTED** | ### Mitigation Details @@ -307,6 +308,15 @@ decompresses transparently before returning chunks). A gzip bomb that decompresses to a large size is caught by the same size limit that protects against unbounded responses (TM-DOS-001). +**TM-DOS-007 — Adversarial PDF parsing (ACCEPTED):** +PDF content processors receive only bodies bounded by `max_body_size`; partial bodies +are rejected before parsing, extracted output is capped again, and synchronous parsing +runs off the async runtime with at most two documents active per process. A crafted PDF +may still cause expensive parser-internal object-stream expansion or prolonged CPU work. +Operators handling hostile, high-volume PDFs should isolate the process and apply outer +request CPU/memory limits. Accepted because the in-process Rust parser has no reliable +cancellation boundary once a blocking parse begins. + ## 5. Information Leakage (TM-LEAK) | ID | Threat | Severity | Mitigation | Status |