[pull] master from buriy:master

.flake8

@@ -1,2 +1,12 @@
 [flake8]
-ignore = E501, W503 
+max-line-length = 127
+max-complexity = 35
+extend-ignore = E501,W503
+per-file-ignores =
+    readability/__init__.py:F401
+exclude =
+    .git,
+    .tox,
+    .venv,
+    build,
+    dist

.github/workflows/python-package.yml

@@ -21,20 +21,8 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install flake8 pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Lint with flake8
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Test with pytest
-      run: |
-        pytest
+    - name: Run pre-release checks
+      run: make pre-release PYTHON_SYSTEM=python

.gitignore

@@ -15,4 +15,5 @@ nosetests.xml
 .cache
 /.noseids
 /.venv
-/poetry.lock
+/poetry.lock
+.DS_Store

Makefile

@@ -1,19 +1,21 @@
 # Makefile to help automate tasks
-WD := $(shell pwd)
-PY := .venv/bin/python
-PIP := .venv/bin/pip
-PEP8 := .venv/bin/pep8
-NOSE := .venv/bin/nosetests
-TWINE := .venv/bin/twine
+PYTHON_SYSTEM ?= python3
+PY := $(CURDIR)/.venv/bin/python
+BENCHMARK_MIN_SCORE ?= 0.95
+VERSION := $(shell sed -n 's/^__version__ = "\(.*\)"/\1/p' readability/__init__.py)
+DIST_FILES := dist/readability_lxml-$(VERSION)-py3-none-any.whl \
+	dist/readability_lxml-$(VERSION).tar.gz
 
 # ###########
 # Tests rule!
 # ###########
 .PHONY: test
-test: venv develop $(NOSE)
-	$(NOSE) --with-id -s tests
+test: develop
+	$(PY) -m pytest -q
 
-$(NOSE): setup
+.PHONY: benchmark
+benchmark: venv develop
+	$(PY) -m readability.benchmark --min-score $(BENCHMARK_MIN_SCORE)
 
 # #######
 # INSTALL
@@ -24,50 +26,57 @@ all: setup develop
 venv: .venv/bin/python
 
 setup: venv
-	$(PIP) install -r requirements-dev.txt | grep -v "already satisfied" || true
+	$(PY) -m pip install -U pip
+	$(PY) -m pip install -r requirements-dev.txt
 
 .venv/bin/python:
-	test -d .venv || which python3 && python3 -m venv .venv || virtualenv .venv
+	test -x .venv/bin/python || $(PYTHON_SYSTEM) -m venv .venv
 
 .PHONY: clean
 clean:
 	rm -rf .venv
 
-develop: .venv/lib/python*/site-packages/readability-lxml.egg-link
-
-.venv/lib/python*/site-packages/readability-lxml.egg-link:
-	$(PY) setup.py develop
+.PHONY: develop
+develop: setup
+	$(PY) -m pip install -e .
 
 
 # ###########
 # Development
 # ###########
 .PHONY: clean_all
-clean_all: clean_venv
+clean_all: clean
+
+.PHONY: lint
+lint: setup
+	$(PY) -m flake8 readability tests benchmarks
+
+.PHONY: check-version
+check-version:
+	test "$(VERSION)" = "$$(sed -n 's/^version = "\(.*\)"/\1/p' pyproject.toml | head -n 1)"
 
 .PHONY: build
 build:
-	poetry build
+	$(PY) -m build
+
+.PHONY: check-dist
+check-dist: setup build
+	$(PY) -m twine check $(DIST_FILES)
+
+.PHONY: check pre-release
+check pre-release: develop lint test benchmark check-version check-dist
 
 # ###########
 # Deploy
 # ###########
 .PHONY: dist
-dist:
-	$(PY) -m pip install wheel
-	$(PY) setup.py sdist bdist_wheel
-	$(TWINE) check dist/*
+dist: check-dist
 
 .PHONY: upload
-upload:
-	$(TWINE) upload dist/*
+upload: check-dist
+	$(PY) -m twine upload $(DIST_FILES)
 
 .PHONY: bump
 bump:
-	$(EDITOR) readability/__init__.py
-	$(eval VERSION := $(shell grep "__version__" readability/__init__.py | cut -d'"' -f2))
-	# fix first occurrence of version in pyproject.toml
-	sed -i '0,/version = ".*"/s//version = "$(VERSION)"/' pyproject.toml
-	git commit -m "Bump version to $(VERSION)" pyproject.toml readability/__init__.py
-	git tag $(VERSION)
-	git push --tags
+	test -n "$(NEW_VERSION)"
+	$(PYTHON_SYSTEM) -c 'from pathlib import Path; files = [(Path("readability/__init__.py"), "__version__ = \"$(VERSION)\"", "__version__ = \"$(NEW_VERSION)\""), (Path("pyproject.toml"), "version = \"$(VERSION)\"", "version = \"$(NEW_VERSION)\"")]; contents = [(path, path.read_text(), old, new) for path, old, new in files]; assert all(old in text for path, text, old, new in contents); [path.write_text(text.replace(old, new, 1)) for path, text, old, new in contents]'

NOTICE

@@ -0,0 +1,23 @@
+python-readability includes test fixtures derived from Mozilla Readability.
+
+Readability
+Copyright (c) 2010 Arc90 Inc
+Copyright (c) 2010-2026 Mozilla and Contributors
+
+Mozilla Readability is licensed under the Apache License, Version 2.0.
+The imported fixture provenance and upstream commit are recorded in each
+pages/*/metadata.json file.
+
+python-readability also includes a fixed sample from dragnet_data.
+
+dragnet_data
+
+The imported Dragnet data is licensed under CC BY 4.0. The full license is
+included at pages/dragnet/LICENSE. Fixture provenance and the pinned upstream
+commit are recorded in each pages/dragnet/*/metadata.json file. Gold text was
+wrapped in HTML paragraphs for the benchmark; source page HTML is unchanged.
+
+python-readability also includes primary saved HTML pages from
+jcharum/lxml-readability commit cec2d35c55cc8b94f0f6ff582cf700dab377af8a.
+The upstream README declares that project under Apache License 2.0. Provenance
+and the generated-baseline method are recorded in pages/jcharum/*/regression.json.

README.md

@@ -28,18 +28,66 @@ $ conda install -c conda-forge readability-lxml
 
 >>> response = requests.get('http://example.com')
 >>> doc = Document(response.content)
->>> doc.title()
-'Example Domain'
+>>> print(doc.title())
+Example Domain
 
->>> doc.summary()
-"""<html><body><div><body id="readabilityBody">\n<div>\n    <h1>Example Domain</h1>\n
+>>> print(doc.summary())
+<html><body><div><body id="readabilityBody">
+<div>
+    <h1>Example Domain</h1>
 <p>This domain is established to be used for illustrative examples in documents. You may
-use this\n    domain in examples without prior coordination or asking for permission.</p>
-\n    <p><a href="http://www.iana.org/domains/example">More information...</a></p>\n</div>
-\n</body>\n</div></body></html>"""
+use this domain in examples without prior coordination or asking for permission.</p>
+    <p><a href="http://www.iana.org/domains/example">More information...</a></p>
+</div>
+</body>
+</div></body></html>
 ```
 
+The command-line interface accepts a local HTML file or a URL:
+
+```bash
+python -m readability -u https://example.com
+```
+
+Run the real-page extraction benchmark with the default minimum F1 of 0.95:
+
+```bash
+make benchmark
+BENCHMARK_MIN_SCORE=0.97 make benchmark
+```
+
+The corpus methodology, version reports, and ten-engine comparison are
+documented in
+[`docs/quality/`](https://github.com/buriy/python-readability/blob/master/docs/quality/README.md).
+
+## Security
+
+`Document.summary()` removes common active content while extracting an article,
+but readability-lxml is not a security boundary or a general-purpose HTML
+sanitizer. Applications that render untrusted output must apply a dedicated
+allowlist sanitizer and an appropriate Content Security Policy.
+
 ## Change Log
+- 0.9
+  - Expanded the declared Python range through 3.14, added `lxml` 6 support, and updated packaging metadata for Markdown documentation and current `cssselect` releases.
+  - Added `python -m readability` execution for local files and URLs.
+  - Fixed bytes input and encoding detection.
+  - Fixed `get_clean_html()` before another API call has initialized the document.
+  - Fixed shortened-title selection and CJK title length handling.
+  - Fixed XPath annotations incorrectly affecting the ruthless-parser retry length.
+  - Preserved inline links and formatting elements when converting misused `<div>` elements into paragraphs.
+  - Removed inline `display: none`, `visibility: hidden`, HTML `hidden`, and `<noscript>` fallback content before scoring.
+  - Preserved content containers whose class or ID also contains an unlikely-candidate term such as `sidebar`.
+  - Preserved code blocks and semantic `<main>` or `<article>` containers during unlikely-candidate filtering.
+  - Recovered editorial leads, heading preambles, split article segments, and substantial list-based articles.
+  - Removed trailing linked calls to action without weakening general link-density filtering.
+  - Restricted retained video iframes to exact HTTP(S) YouTube and Vimeo hosts, preventing lookalike-host and userinfo URL bypasses, and removed active `srcdoc` content.
+  - Added 181 reproducible fixtures: 166 quality fixtures split into base, Dragnet, GitHub user-issue, and complete 130-page Mozilla Readability corpora, plus 15 manually curated real-page regression fixtures from `jcharum/lxml-readability`.
+  - Replaced the unmaintained nose test runner with pytest in local, tox, and GitHub Actions workflows.
+  - Updated development and release targets for portable module execution, PEP 517 builds, version synchronization, isolated artifact checks, and current-version uploads.
+  - Improved the 166-page benchmark from precision 0.971, recall 0.884, and F1 0.926 in 0.8.4.1 to precision 0.991, recall 0.956, and F1 0.973.
+  - Corrected the README usage examples.
+  - Fixes GitHub issues [#14](https://github.com/buriy/python-readability/issues/14), [#108](https://github.com/buriy/python-readability/issues/108), [#119](https://github.com/buriy/python-readability/issues/119), [#130](https://github.com/buriy/python-readability/issues/130), [#143](https://github.com/buriy/python-readability/issues/143), [#146](https://github.com/buriy/python-readability/issues/146), [#153](https://github.com/buriy/python-readability/issues/153), [#158](https://github.com/buriy/python-readability/issues/158), [#159](https://github.com/buriy/python-readability/issues/159), [#163](https://github.com/buriy/python-readability/issues/163), [#170](https://github.com/buriy/python-readability/issues/170), [#176](https://github.com/buriy/python-readability/issues/176), [#182](https://github.com/buriy/python-readability/issues/182), and [#194](https://github.com/buriy/python-readability/issues/194). Release tracking issue [#196](https://github.com/buriy/python-readability/issues/196) can be closed after 0.9 is published to PyPI.
 - 0.8.4 Better CJK support, thanks @cdhigh
 - 0.8.3.1 Support for python 3.8 - 3.13
 - 0.8.3 We can now save all images via keep_all_images=True (default is to save 1 main image), thanks @botlabsDev

benchmarks/arc90_js.mjs

@@ -0,0 +1,54 @@
+import { createRequire } from "node:module";
+import fs from "node:fs";
+import path from "node:path";
+import process from "node:process";
+import readline from "node:readline";
+import vm from "node:vm";
+import { pathToFileURL } from "node:url";
+
+
+const environment = path.resolve(process.argv[2]);
+const require = createRequire(pathToFileURL(path.join(environment, "package.json")));
+const { JSDOM, VirtualConsole } = require("jsdom");
+const sourcePath = path.join(environment, "readability.js");
+const readabilityScript = new vm.Script(fs.readFileSync(sourcePath, "utf8"), {
+  filename: sourcePath,
+});
+
+const virtualConsole = new VirtualConsole();
+virtualConsole.on("jsdomError", error => {
+  if (error.type !== "css parsing") {
+    process.stderr.write(error.stack + "\n");
+  }
+});
+
+const lines = readline.createInterface({ input: process.stdin });
+for await (const line of lines) {
+  let dom;
+  try {
+    const request = JSON.parse(line);
+    dom = new JSDOM(request.source, {
+      url: request.url,
+      runScripts: "outside-only",
+      virtualConsole,
+    });
+    const window = dom.window;
+    window.readConvertLinksToFootnotes = false;
+    window.readStyle = "style-novel";
+    window.readSize = "size-medium";
+    window.readMargin = "margin-wide";
+    window.scrollTo = () => {};
+    for (const stylesheet of window.document.styleSheets) {
+      if (stylesheet.href === undefined) {
+        Object.defineProperty(stylesheet, "href", { value: null });
+      }
+    }
+    readabilityScript.runInContext(dom.getInternalVMContext());
+    const article = window.document.querySelector("#readability-content");
+    process.stdout.write(JSON.stringify({ content: article.innerHTML }) + "\n");
+  } catch (error) {
+    process.stdout.write(JSON.stringify({ error: error.stack }) + "\n");
+  } finally {
+    dom?.window.close();
+  }
+}