Add furst-read, an article extractor and minimal renderer
Fetches a page, discards everything that is not the article, and renders what is left as a self-contained HTML document with no scripts, no stylesheets, no webfonts and no third-party requests. The reader rule in the starter config already points at it. Extraction follows Readability: score p/pre/td/blockquote by prose weight, propagate to ancestors with decay, adjust by class and id names, scale by one minus link density, then take the winner plus sibling nodes that also read like body copy. Serialisation runs against a tag whitelist, with unlisted elements contributing their children but no tag of their own, so wrapper divs disappear. Handling for what real pages actually do: - follow meta http-equiv=refresh stubs, including inside noscript - fall back to data-src when src holds a lazy-load placeholder - take the first srcset candidate, the smallest, not the last - decode via Content-Type charset, then meta charset, then UTF-8 - resolve links against the post-redirect URL so file:// output works - ignore script and style text so a page cannot inflate its own score Output goes to a cache file named by a hash of the URL and opens in $FURST_BROWSER, $BROWSER, or the first light browser on $PATH; --html, --text, --out and --stdin cover the other uses. Restructures the repository as a workspace so the router keeps its two dependencies and its fast build.
This commit is contained in:
parent
ec79c07b0f
commit
6dcb00da80
746
Cargo.lock
generated
746
Cargo.lock
generated
@ -2,12 +2,148 @@
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "adler2"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cssparser"
|
||||
version = "0.34.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7c66d1cd8ed61bf80b38432613a7a2f09401ab8d0501110655f8b341484a3e3"
|
||||
dependencies = [
|
||||
"cssparser-macros",
|
||||
"dtoa-short",
|
||||
"itoa",
|
||||
"phf",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cssparser-macros"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_more"
|
||||
version = "0.99.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dtoa"
|
||||
version = "1.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590"
|
||||
|
||||
[[package]]
|
||||
name = "dtoa-short"
|
||||
version = "0.3.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87"
|
||||
dependencies = [
|
||||
"dtoa",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ego-tree"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2972feb8dffe7bc8c5463b1dacda1b0dfbed3710e50f977d965429692d74cd8"
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
"zlib-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "furst"
|
||||
version = "0.1.0"
|
||||
@ -16,12 +152,89 @@ dependencies = [
|
||||
"toml",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "furst-read"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"ego-tree",
|
||||
"encoding_rs",
|
||||
"scraper",
|
||||
"ureq",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futf"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"
|
||||
dependencies = [
|
||||
"mac",
|
||||
"new_debug_unreachable",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fxhash"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getopts"
|
||||
version = "0.2.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df"
|
||||
dependencies = [
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.17.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
|
||||
|
||||
[[package]]
|
||||
name = "html5ever"
|
||||
version = "0.29.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b7410cae13cbc75623c98ac4cbfd1f0bedddf3227afc24f370cf0f50a44a11c"
|
||||
dependencies = [
|
||||
"log",
|
||||
"mac",
|
||||
"markup5ever",
|
||||
"match_token",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"itoa",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "httparse"
|
||||
version = "1.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.14.2"
|
||||
@ -32,12 +245,179 @@ dependencies = [
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.189"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
|
||||
dependencies = [
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
|
||||
|
||||
[[package]]
|
||||
name = "mac"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "markup5ever"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c7a7213d12e1864c0f002f52c2923d4556935a43dec5e71355c2760e0f6e7a18"
|
||||
dependencies = [
|
||||
"log",
|
||||
"phf",
|
||||
"phf_codegen",
|
||||
"string_cache",
|
||||
"string_cache_codegen",
|
||||
"tendril",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "match_token"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "88a9689d8d44bf9964484516275f5cd4c9b59457a6940c1d5d0ecbb94510a36b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
|
||||
dependencies = [
|
||||
"adler2",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "new_debug_unreachable"
|
||||
version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.12.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
|
||||
dependencies = [
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.9.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"smallvec",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
||||
|
||||
[[package]]
|
||||
name = "phf"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
|
||||
dependencies = [
|
||||
"phf_macros",
|
||||
"phf_shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_codegen"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"
|
||||
dependencies = [
|
||||
"phf_generator",
|
||||
"phf_shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
|
||||
dependencies = [
|
||||
"phf_shared",
|
||||
"rand",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_macros"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216"
|
||||
dependencies = [
|
||||
"phf_generator",
|
||||
"phf_shared",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
|
||||
dependencies = [
|
||||
"siphasher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "precomputed-hash"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.107"
|
||||
@ -56,6 +436,119 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c"
|
||||
dependencies = [
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.5.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ring"
|
||||
version = "0.17.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"getrandom",
|
||||
"libc",
|
||||
"untrusted",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.23.43"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06"
|
||||
dependencies = [
|
||||
"log",
|
||||
"once_cell",
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"rustls-webpki",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-pki-types"
|
||||
version = "1.15.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96"
|
||||
dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.103.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
|
||||
dependencies = [
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"untrusted",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "scraper"
|
||||
version = "0.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "527e65d9d888567588db4c12da1087598d0f6f8b346cc2c5abc91f05fc2dffe2"
|
||||
dependencies = [
|
||||
"cssparser",
|
||||
"ego-tree",
|
||||
"getopts",
|
||||
"html5ever",
|
||||
"precomputed-hash",
|
||||
"selectors",
|
||||
"tendril",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "selectors"
|
||||
version = "0.26.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cssparser",
|
||||
"derive_more",
|
||||
"fxhash",
|
||||
"log",
|
||||
"new_debug_unreachable",
|
||||
"phf",
|
||||
"phf_codegen",
|
||||
"precomputed-hash",
|
||||
"servo_arc",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.229"
|
||||
@ -83,7 +576,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -95,6 +588,87 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "servo_arc"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930"
|
||||
dependencies = [
|
||||
"stable_deref_trait",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649"
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f"
|
||||
|
||||
[[package]]
|
||||
name = "stable_deref_trait"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
|
||||
|
||||
[[package]]
|
||||
name = "string_cache"
|
||||
version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f"
|
||||
dependencies = [
|
||||
"new_debug_unreachable",
|
||||
"parking_lot",
|
||||
"phf_shared",
|
||||
"precomputed-hash",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "string_cache_codegen"
|
||||
version = "0.5.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0"
|
||||
dependencies = [
|
||||
"phf_generator",
|
||||
"phf_shared",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "subtle"
|
||||
version = "2.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.119"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.5"
|
||||
@ -106,6 +680,17 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tendril"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0"
|
||||
dependencies = [
|
||||
"futf",
|
||||
"mac",
|
||||
"utf-8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml"
|
||||
version = "0.8.23"
|
||||
@ -146,6 +731,153 @@ version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
|
||||
|
||||
[[package]]
|
||||
name = "ureq"
|
||||
version = "3.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af5546be8f5378d5414f83733f5c9a2526f4645829edbc1c41790aeef1b38e8b"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"flate2",
|
||||
"log",
|
||||
"percent-encoding",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"ureq-proto",
|
||||
"utf8-zero",
|
||||
"webpki-roots",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ureq-proto"
|
||||
version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fabc3e92916c89c95b20eef7b06b00b066bc217ef9ea3a4ac9bf1a7e35261e10"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"http",
|
||||
"httparse",
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf-8"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
|
||||
|
||||
[[package]]
|
||||
name = "utf8-zero"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.1+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "1.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.52.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
|
||||
dependencies = [
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
|
||||
dependencies = [
|
||||
"windows_aarch64_gnullvm",
|
||||
"windows_aarch64_msvc",
|
||||
"windows_i686_gnu",
|
||||
"windows_i686_gnullvm",
|
||||
"windows_i686_msvc",
|
||||
"windows_x86_64_gnu",
|
||||
"windows_x86_64_gnullvm",
|
||||
"windows_x86_64_msvc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||
|
||||
[[package]]
|
||||
name = "winnow"
|
||||
version = "0.7.15"
|
||||
@ -154,3 +886,15 @@ checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zeroize"
|
||||
version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
|
||||
|
||||
[[package]]
|
||||
name = "zlib-rs"
|
||||
version = "0.6.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12"
|
||||
|
||||
@ -1,3 +1,6 @@
|
||||
[workspace]
|
||||
members = ["furst-read"]
|
||||
|
||||
[package]
|
||||
name = "furst"
|
||||
version = "0.1.0"
|
||||
|
||||
@ -97,6 +97,12 @@ If no argument mentions `{url}` or `{url_enc}`, the URL is appended last.
|
||||
- `furst` `exec`s the handler rather than forking, so it leaves no process
|
||||
behind.
|
||||
|
||||
## Companion tools
|
||||
|
||||
- [`furst-read`](furst-read/README.md) — fetch a page, extract the article, and
|
||||
render it as minimal HTML with no scripts, stylesheets, or webfonts. The
|
||||
`reader` rule in the starter config points at it.
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
|
||||
14
furst-read/Cargo.toml
Normal file
14
furst-read/Cargo.toml
Normal file
@ -0,0 +1,14 @@
|
||||
[package]
|
||||
name = "furst-read"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
description = "Strip a web page down to the article and render it as minimal HTML"
|
||||
|
||||
[dependencies]
|
||||
ureq = "3"
|
||||
scraper = "0.23"
|
||||
encoding_rs = "0.8"
|
||||
ego-tree = "0.10.0"
|
||||
|
||||
[profile.release]
|
||||
opt-level = "s"
|
||||
74
furst-read/README.md
Normal file
74
furst-read/README.md
Normal file
@ -0,0 +1,74 @@
|
||||
# furst-read
|
||||
|
||||
Fetch a page, throw away everything that is not the article, and render what is
|
||||
left as a small self-contained HTML document — no scripts, no stylesheets, no
|
||||
webfonts, no third-party requests.
|
||||
|
||||
This is the part of the toolset that does the real work. Choosing a lighter
|
||||
browser engine buys 2–3×. Not loading 3MB of JavaScript buys considerably more.
|
||||
|
||||
## Use
|
||||
|
||||
```sh
|
||||
furst-read <url> # render and open in a browser (default)
|
||||
furst-read --html <url> # minimal HTML to stdout
|
||||
furst-read --text <url> # plain text to stdout
|
||||
furst-read --out page.html <url>
|
||||
furst-read --no-images <url>
|
||||
curl -s <url> | furst-read --stdin --text <url>
|
||||
```
|
||||
|
||||
The default writes to `$XDG_CACHE_HOME/furst-read/<hash>.html` and opens it in
|
||||
`$FURST_BROWSER`, `$BROWSER`, or the first light browser on `$PATH`. Because the
|
||||
file is named by a hash of the URL, revisiting a page reuses it instead of
|
||||
littering.
|
||||
|
||||
Slots straight into [furst](../README.md):
|
||||
|
||||
```toml
|
||||
[[rule]]
|
||||
name = "reader"
|
||||
hosts = ["wikipedia.org", "news.ycombinator.com", "*.substack.com"]
|
||||
run = ["furst-read", "{url}"]
|
||||
```
|
||||
|
||||
## How it finds the article
|
||||
|
||||
Readability's approach, reimplemented:
|
||||
|
||||
1. Score `p`, `pre`, `td` and `blockquote` by how much prose they hold —
|
||||
character count and comma count, which boilerplate has neither of.
|
||||
2. Push each score up to the ancestors: parent takes it whole, grandparent
|
||||
half, and it thins out from there.
|
||||
3. Weight by `class` and `id`. `post-content` gains, `sidebar` loses.
|
||||
4. Multiply by `1 − linkDensity`, which is what separates a nav block from a
|
||||
paragraph.
|
||||
5. Take the winner, then pull in sibling nodes that also read like body copy,
|
||||
because articles are routinely split across several divs.
|
||||
|
||||
Then serialise against a tag whitelist. Wrapper elements contribute their
|
||||
children and no tag of their own, which is how the div soup disappears.
|
||||
|
||||
## Details that matter in practice
|
||||
|
||||
- **Redirect stubs.** A `<meta http-equiv="refresh">` is followed, including
|
||||
when it is tucked inside `<noscript>`, which is how most static site
|
||||
generators write them.
|
||||
- **Lazy images.** A `src` holding a placeholder falls back to `data-src` and
|
||||
friends, so the images are the real ones.
|
||||
- **`srcset` takes the first candidate, not the last** — on old hardware the
|
||||
smallest variant is the one you want.
|
||||
- **Character sets.** Content-Type first, then `<meta charset>`, then UTF-8.
|
||||
Latin-1 pages are still common enough to matter.
|
||||
- **Links are made absolute** against the post-redirect URL, so the output
|
||||
works from a `file://` path.
|
||||
- Scripts and styles are never counted as text, so a page cannot inflate its
|
||||
own score with an inline bundle.
|
||||
|
||||
## Limits
|
||||
|
||||
- Listing pages and forums have no single article to find. The extractor will
|
||||
return whatever scored highest, which may be very little.
|
||||
- Anything that renders client-side arrives empty; there is no JavaScript here
|
||||
and that is the point.
|
||||
- Paywalls and interstitials are not defeated, only rendered plainly.
|
||||
391
furst-read/src/extract.rs
Normal file
391
furst-read/src/extract.rs
Normal file
@ -0,0 +1,391 @@
|
||||
//! Find the article in a page.
|
||||
//!
|
||||
//! This is the Readability approach: score block elements by how much prose
|
||||
//! they hold, push that score up to their ancestors, weight by class and id
|
||||
//! names, penalise link density, then take the winner plus any siblings that
|
||||
//! also look like body copy. It gets the large majority of real articles right
|
||||
//! and stays small enough to tune.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use ego_tree::{NodeId, NodeRef};
|
||||
use scraper::{ElementRef, Html, Node, Selector};
|
||||
|
||||
use crate::urljoin::join;
|
||||
|
||||
/// Container names that never hold the article body.
|
||||
const SKIP: &[&str] = &[
|
||||
"script", "style", "noscript", "iframe", "svg", "canvas", "form", "nav", "aside", "footer",
|
||||
"header", "button", "select", "textarea", "template", "object", "embed",
|
||||
];
|
||||
|
||||
const POSITIVE: &[&str] =
|
||||
&["article", "body", "content", "entry", "hentry", "h-entry", "main", "page", "post", "text", "blog", "story"];
|
||||
|
||||
const NEGATIVE: &[&str] = &[
|
||||
"comment", "combx", "contact", "foot", "masthead", "meta", "outbrain", "promo", "related",
|
||||
"scroll", "share", "shoutbox", "sidebar", "sponsor", "shopping", "tags", "widget", "banner",
|
||||
"ad-", "adbox", "advert", "social", "nav", "menu", "modal", "popup", "cookie", "newsletter",
|
||||
"subscribe", "byline", "author", "breadcrumb",
|
||||
];
|
||||
|
||||
static SEL_A: LazyLock<Selector> = LazyLock::new(|| Selector::parse("a").unwrap());
|
||||
static SEL_META: LazyLock<Selector> = LazyLock::new(|| Selector::parse("meta").unwrap());
|
||||
static SEL_BLOCK: LazyLock<Selector> = LazyLock::new(|| Selector::parse("p, pre, td, blockquote").unwrap());
|
||||
|
||||
pub struct Article {
|
||||
pub title: String,
|
||||
pub byline: Option<String>,
|
||||
pub site: Option<String>,
|
||||
/// Top-level nodes of the extracted body, in document order.
|
||||
pub nodes: Vec<NodeId>,
|
||||
}
|
||||
|
||||
pub fn extract(doc: &Html) -> Article {
|
||||
let site = meta_content(doc, &["og:site_name", "application-name"]);
|
||||
Article {
|
||||
title: title(doc, site.as_deref()),
|
||||
byline: meta_content(doc, &["author", "article:author", "og:article:author", "twitter:creator"]),
|
||||
site,
|
||||
nodes: body(doc),
|
||||
}
|
||||
}
|
||||
|
||||
fn body(doc: &Html) -> Vec<NodeId> {
|
||||
let mut scores: HashMap<NodeId, f64> = HashMap::new();
|
||||
|
||||
for el in doc.select(&SEL_BLOCK) {
|
||||
if under_skipped(el) {
|
||||
continue;
|
||||
}
|
||||
let text = inner_text(el);
|
||||
let text = text.trim();
|
||||
let len = text.chars().count();
|
||||
if len < 25 {
|
||||
continue;
|
||||
}
|
||||
// Prose has commas and length; navigation and boilerplate have neither.
|
||||
let score = 1.0 + text.matches(',').count() as f64 + (len as f64 / 100.0).floor().min(3.0);
|
||||
|
||||
for (level, anc) in el.ancestors().enumerate() {
|
||||
let Some(ae) = ElementRef::wrap(anc) else { continue };
|
||||
if ae.value().name() == "html" {
|
||||
break;
|
||||
}
|
||||
// Parent takes it all, grandparent half, then it thins out fast.
|
||||
let divisor = match level {
|
||||
0 => 1.0,
|
||||
1 => 2.0,
|
||||
n => n as f64 * 3.0,
|
||||
};
|
||||
*scores.entry(anc.id()).or_insert(0.0) += score / divisor;
|
||||
if level >= 3 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut best: Option<(NodeId, f64)> = None;
|
||||
for (&id, &raw) in &scores {
|
||||
let Some(node) = doc.tree.get(id) else { continue };
|
||||
let Some(el) = ElementRef::wrap(node) else { continue };
|
||||
if matches!(el.value().name(), "body" | "html") {
|
||||
continue;
|
||||
}
|
||||
let score = (raw + class_weight(el)) * (1.0 - link_density(el));
|
||||
if best.is_none_or(|(_, b)| score > b) {
|
||||
best = Some((id, score));
|
||||
}
|
||||
}
|
||||
|
||||
let Some((top_id, top_score)) = best else {
|
||||
return fallback(doc);
|
||||
};
|
||||
let Some(top) = doc.tree.get(top_id) else {
|
||||
return fallback(doc);
|
||||
};
|
||||
|
||||
// An article is often split across sibling divs. Pull in the ones that also
|
||||
// read like body copy, so we do not return only the first half.
|
||||
let parent = top.parent().filter(|p| ElementRef::wrap(*p).is_some_and(|e| e.value().name() != "html"));
|
||||
let Some(parent) = parent else {
|
||||
return vec![top_id];
|
||||
};
|
||||
|
||||
let threshold = f64::max(10.0, top_score * 0.2);
|
||||
let mut out = Vec::new();
|
||||
for sib in parent.children() {
|
||||
if sib.id() == top_id {
|
||||
out.push(top_id);
|
||||
continue;
|
||||
}
|
||||
let Some(se) = ElementRef::wrap(sib) else { continue };
|
||||
if SKIP.contains(&se.value().name()) {
|
||||
continue;
|
||||
}
|
||||
let raw = scores.get(&sib.id()).copied().unwrap_or(0.0);
|
||||
if (raw + class_weight(se)) * (1.0 - link_density(se)) >= threshold {
|
||||
out.push(sib.id());
|
||||
continue;
|
||||
}
|
||||
// A bare <p> next to the winner is almost always part of the article.
|
||||
if se.value().name() == "p" {
|
||||
let text = inner_text(se);
|
||||
let text = text.trim();
|
||||
let len = text.chars().count();
|
||||
let density = link_density(se);
|
||||
let looks_like_prose = (len > 80 && density < 0.25)
|
||||
|| (len > 0 && len <= 80 && density == 0.0 && text.ends_with(['.', '!', '?', '。', '！', '？']));
|
||||
if looks_like_prose {
|
||||
out.push(sib.id());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if out.is_empty() { vec![top_id] } else { out }
|
||||
}
|
||||
|
||||
fn fallback(doc: &Html) -> Vec<NodeId> {
|
||||
for pattern in ["article", "main", "[role=\"main\"]", "body"] {
|
||||
if let Ok(sel) = Selector::parse(pattern)
|
||||
&& let Some(el) = doc.select(&sel).next()
|
||||
{
|
||||
return vec![el.id()];
|
||||
}
|
||||
}
|
||||
vec![doc.tree.root().id()]
|
||||
}
|
||||
|
||||
fn title(doc: &Html, site: Option<&str>) -> String {
|
||||
// og:title is authoritative but is often still "Article | Site".
|
||||
if let Some(t) = meta_content(doc, &["og:title", "twitter:title"]) {
|
||||
return strip_site_suffix(&t, site);
|
||||
}
|
||||
if let Some(h) = first_text(doc, "h1")
|
||||
&& !h.is_empty()
|
||||
{
|
||||
return h;
|
||||
}
|
||||
first_text(doc, "title").map(|t| strip_site_suffix(&t, site)).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// "Article name | Site Name" -> "Article name". Strip when the tail is the
|
||||
/// site's own name, or failing that when it is a short fragment shorter than
|
||||
/// what precedes it — a title with a real dash in it must survive.
|
||||
fn strip_site_suffix(t: &str, site: Option<&str>) -> String {
|
||||
let t = t.trim();
|
||||
for sep in [" | ", " - ", " — ", " – ", " · ", " :: ", " » ", " • "] {
|
||||
let Some(i) = t.rfind(sep) else { continue };
|
||||
let head = t[..i].trim();
|
||||
let tail = t[i + sep.len()..].trim();
|
||||
if head.is_empty() || tail.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if site.is_some_and(|s| tail.eq_ignore_ascii_case(s.trim())) {
|
||||
return head.to_string();
|
||||
}
|
||||
if tail.split_whitespace().count() <= 5 && head.chars().count() > tail.chars().count() {
|
||||
return head.to_string();
|
||||
}
|
||||
}
|
||||
t.to_string()
|
||||
}
|
||||
|
||||
fn first_text(doc: &Html, pattern: &str) -> Option<String> {
|
||||
let sel = Selector::parse(pattern).ok()?;
|
||||
let el = doc.select(&sel).next()?;
|
||||
Some(normalize(&inner_text(el)))
|
||||
}
|
||||
|
||||
fn meta_content(doc: &Html, keys: &[&str]) -> Option<String> {
|
||||
for el in doc.select(&SEL_META) {
|
||||
let Some(key) = el.value().attr("property").or_else(|| el.value().attr("name")) else {
|
||||
continue;
|
||||
};
|
||||
if !keys.iter().any(|k| k.eq_ignore_ascii_case(key)) {
|
||||
continue;
|
||||
}
|
||||
let value = normalize(el.value().attr("content").unwrap_or(""));
|
||||
if !value.is_empty() {
|
||||
return Some(value);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Text of an element, ignoring subtrees that are not prose. `ElementRef::text`
|
||||
/// would happily hand back the contents of a `<script>`.
|
||||
pub fn inner_text(el: ElementRef) -> String {
|
||||
fn walk(node: NodeRef<Node>, out: &mut String) {
|
||||
for child in node.children() {
|
||||
match child.value() {
|
||||
Node::Text(t) => out.push_str(&t.text),
|
||||
Node::Element(e) if !matches!(e.name(), "script" | "style" | "noscript" | "template") => {
|
||||
walk(child, out)
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut out = String::new();
|
||||
walk(*el, &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
pub fn link_density(el: ElementRef) -> f64 {
|
||||
let total = inner_text(el).chars().count() as f64;
|
||||
if total == 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
let linked: f64 = el.select(&SEL_A).map(|a| inner_text(a).chars().count() as f64).sum();
|
||||
(linked / total).min(1.0)
|
||||
}
|
||||
|
||||
fn class_weight(el: ElementRef) -> f64 {
|
||||
let id = el.value().id().unwrap_or_default().to_ascii_lowercase();
|
||||
let class = el.value().attr("class").unwrap_or_default().to_ascii_lowercase();
|
||||
let mut weight = 0.0;
|
||||
for haystack in [&id, &class] {
|
||||
if NEGATIVE.iter().any(|n| haystack.contains(n)) {
|
||||
weight -= 25.0;
|
||||
}
|
||||
if POSITIVE.iter().any(|p| haystack.contains(p)) {
|
||||
weight += 25.0;
|
||||
}
|
||||
}
|
||||
weight
|
||||
}
|
||||
|
||||
fn under_skipped(el: ElementRef) -> bool {
|
||||
el.ancestors()
|
||||
.filter_map(ElementRef::wrap)
|
||||
.any(|a| SKIP.contains(&a.value().name()))
|
||||
}
|
||||
|
||||
pub fn normalize(s: &str) -> String {
|
||||
s.split_whitespace().collect::<Vec<_>>().join(" ")
|
||||
}
|
||||
|
||||
/// Follow `<meta http-equiv="refresh">`. Static site generators use these as
|
||||
/// redirect stubs, and they are often tucked inside `<noscript>`, so fall back
|
||||
/// to reparsing that content when the selector alone finds nothing.
|
||||
pub fn meta_refresh(doc: &Html, base: &str) -> Option<String> {
|
||||
if let Some(url) = refresh_target(doc) {
|
||||
return Some(join(base, &url));
|
||||
}
|
||||
static SEL_NOSCRIPT: LazyLock<Selector> = LazyLock::new(|| Selector::parse("noscript").unwrap());
|
||||
for el in doc.select(&SEL_NOSCRIPT) {
|
||||
let inner = Html::parse_fragment(&el.text().collect::<String>());
|
||||
if let Some(url) = refresh_target(&inner) {
|
||||
return Some(join(base, &url));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn refresh_target(doc: &Html) -> Option<String> {
|
||||
for el in doc.select(&SEL_META) {
|
||||
let Some(equiv) = el.value().attr("http-equiv") else { continue };
|
||||
if !equiv.trim().eq_ignore_ascii_case("refresh") {
|
||||
continue;
|
||||
}
|
||||
let Some(content) = el.value().attr("content") else { continue };
|
||||
// content is `<seconds>; url=<target>`
|
||||
let Some(i) = content.to_ascii_lowercase().find("url=") else { continue };
|
||||
let target = content[i + 4..].trim().trim_matches(['"', '\'']);
|
||||
if !target.is_empty() {
|
||||
return Some(target.to_string());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const PAGE: &str = r#"
|
||||
<html><head><title>Real Title | Some Site</title>
|
||||
<meta name="author" content="A Writer"></head>
|
||||
<body>
|
||||
<nav><a href="/1">Home</a><a href="/2">About</a><a href="/3">Contact</a></nav>
|
||||
<div id="sidebar"><p>Subscribe to our newsletter for more content today ok</p></div>
|
||||
<div class="post-content">
|
||||
<p>The first paragraph of the article, which is long enough to score, and has commas.</p>
|
||||
<p>A second paragraph, also long enough to be counted as real prose by the scorer.</p>
|
||||
<script>var junk = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";</script>
|
||||
</div>
|
||||
<footer><p>Copyright notice that is long enough to be scored, with a comma.</p></footer>
|
||||
</body></html>"#;
|
||||
|
||||
fn doc() -> Html {
|
||||
Html::parse_document(PAGE)
|
||||
}
|
||||
|
||||
fn extracted_text(d: &Html) -> String {
|
||||
extract(d)
|
||||
.nodes
|
||||
.iter()
|
||||
.filter_map(|id| d.tree.get(*id))
|
||||
.filter_map(ElementRef::wrap)
|
||||
.map(inner_text)
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn picks_the_content_container_over_boilerplate() {
|
||||
let d = doc();
|
||||
let text = extracted_text(&d);
|
||||
assert!(text.contains("first paragraph of the article"));
|
||||
assert!(text.contains("second paragraph"));
|
||||
assert!(!text.contains("Copyright notice"));
|
||||
assert!(!text.contains("newsletter"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_title_with_a_real_dash_survives() {
|
||||
assert_eq!(
|
||||
strip_site_suffix("Rust 1.81 - What's New in This Release", None),
|
||||
"Rust 1.81 - What's New in This Release"
|
||||
);
|
||||
// ...but a declared site name is always stripped.
|
||||
assert_eq!(strip_site_suffix("Ownership - The Rust Book", Some("The Rust Book")), "Ownership");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn title_loses_the_site_suffix_and_byline_is_found() {
|
||||
let art = extract(&doc());
|
||||
assert_eq!(art.title, "Real Title");
|
||||
assert_eq!(art.byline.as_deref(), Some("A Writer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn script_bodies_are_not_counted_as_text() {
|
||||
let d = doc();
|
||||
let el = d.select(&Selector::parse("div.post-content").unwrap()).next().unwrap();
|
||||
assert!(!inner_text(el).contains("var junk"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn link_density_separates_navigation_from_prose() {
|
||||
let d = doc();
|
||||
let nav = d.select(&Selector::parse("nav").unwrap()).next().unwrap();
|
||||
let body = d.select(&Selector::parse("div.post-content").unwrap()).next().unwrap();
|
||||
assert_eq!(link_density(nav), 1.0);
|
||||
assert_eq!(link_density(body), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn follows_a_meta_refresh_stub_inside_noscript() {
|
||||
let d = Html::parse_document(
|
||||
r#"<html><head><noscript>
|
||||
<meta http-equiv="refresh" content="0; url=/next/page.html">
|
||||
</noscript></head><body><p>go</p></body></html>"#,
|
||||
);
|
||||
assert_eq!(
|
||||
meta_refresh(&d, "https://ex.test/a/b.html").as_deref(),
|
||||
Some("https://ex.test/next/page.html")
|
||||
);
|
||||
}
|
||||
}
|
||||
93
furst-read/src/fetch.rs
Normal file
93
furst-read/src/fetch.rs
Normal file
@ -0,0 +1,93 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use ureq::ResponseExt;
|
||||
|
||||
/// Sites routinely serve a degraded page or a block to unknown clients. This is
|
||||
/// the user's own browsing, taken out of the browser; present it as such.
|
||||
const UA: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0";
|
||||
|
||||
pub struct Page {
|
||||
/// The URL after redirects — the correct base for resolving links.
|
||||
pub url: String,
|
||||
pub html: String,
|
||||
}
|
||||
|
||||
pub fn get(url: &str, limit: usize) -> Result<Page, String> {
|
||||
let agent: ureq::Agent = ureq::Agent::config_builder()
|
||||
.max_redirects(8)
|
||||
.timeout_global(Some(Duration::from_secs(25)))
|
||||
.user_agent(UA)
|
||||
.build()
|
||||
.into();
|
||||
|
||||
let mut resp = agent.get(url).call().map_err(|e| format!("{url}: {e}"))?;
|
||||
let final_url = resp.get_uri().to_string();
|
||||
let content_type = resp
|
||||
.headers()
|
||||
.get("content-type")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let bytes = resp
|
||||
.body_mut()
|
||||
.with_config()
|
||||
.limit(limit as u64)
|
||||
.read_to_vec()
|
||||
.map_err(|e| format!("{url}: {e}"))?;
|
||||
|
||||
Ok(Page { url: final_url, html: decode(&bytes, &content_type) })
|
||||
}
|
||||
|
||||
/// Decode to text using, in order: the Content-Type charset, a `<meta charset>`
|
||||
/// in the head, then UTF-8. Latin-1 pages are still common enough to matter.
|
||||
pub fn decode(bytes: &[u8], content_type: &str) -> String {
|
||||
let label = charset_of(content_type)
|
||||
.or_else(|| meta_charset(bytes))
|
||||
.unwrap_or_else(|| "utf-8".to_string());
|
||||
let enc = encoding_rs::Encoding::for_label(label.trim().as_bytes()).unwrap_or(encoding_rs::UTF_8);
|
||||
enc.decode(bytes).0.into_owned()
|
||||
}
|
||||
|
||||
fn charset_of(content_type: &str) -> Option<String> {
|
||||
let lower = content_type.to_ascii_lowercase();
|
||||
let i = lower.find("charset=")? + "charset=".len();
|
||||
let rest = &lower[i..];
|
||||
let end = rest.find([';', ' ']).unwrap_or(rest.len());
|
||||
Some(rest[..end].trim_matches(['"', '\'']).to_string())
|
||||
}
|
||||
|
||||
fn meta_charset(bytes: &[u8]) -> Option<String> {
|
||||
// The declaration is required to be in the first 1024 bytes.
|
||||
let head = String::from_utf8_lossy(&bytes[..bytes.len().min(2048)]).to_ascii_lowercase();
|
||||
let i = head.find("charset=")? + "charset=".len();
|
||||
let rest = head[i..].trim_start_matches(['"', '\'', ' ']);
|
||||
let end = rest.find(['"', '\'', ' ', '>', ';', '/', '\n', '\r', '\t']).unwrap_or(rest.len());
|
||||
(end > 0).then(|| rest[..end].to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn charset_comes_from_the_header_first() {
|
||||
assert_eq!(charset_of("text/html; charset=ISO-8859-1"), Some("iso-8859-1".into()));
|
||||
assert_eq!(charset_of("text/html"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn falls_back_to_the_meta_declaration() {
|
||||
let b = br#"<html><head><meta charset="windows-1252"></head>"#;
|
||||
assert_eq!(meta_charset(b), Some("windows-1252".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_latin1_when_declared() {
|
||||
// 0xE9 is é in latin-1 and invalid UTF-8.
|
||||
let bytes = [b'c', b'a', b'f', 0xE9];
|
||||
assert_eq!(decode(&bytes, "text/html; charset=iso-8859-1"), "café");
|
||||
// Without a declaration it must not panic, only substitute.
|
||||
assert!(decode(&bytes, "text/html").starts_with("caf"));
|
||||
}
|
||||
}
|
||||
164
furst-read/src/main.rs
Normal file
164
furst-read/src/main.rs
Normal file
@ -0,0 +1,164 @@
|
||||
mod extract;
|
||||
mod fetch;
|
||||
mod render;
|
||||
mod urljoin;
|
||||
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::io::{self, Read, Write};
|
||||
use std::os::unix::process::CommandExt;
|
||||
use std::path::PathBuf;
|
||||
use std::process::{Command, ExitCode};
|
||||
|
||||
const USAGE: &str = "\
|
||||
furst-read — strip a page down to the article and render it as minimal HTML
|
||||
|
||||
USAGE
|
||||
furst-read [OPTIONS] <url>
|
||||
|
||||
OPTIONS
|
||||
--html write the minimal HTML to stdout
|
||||
--text write plain text to stdout
|
||||
--out <file> write the HTML to a file
|
||||
--open render, then open in a browser (default)
|
||||
--browser <cmd> browser to open with
|
||||
--no-images drop images entirely
|
||||
--stdin read the HTML from stdin; <url> is used only as the link base
|
||||
|
||||
With none of --html/--text/--out, the page is cached under
|
||||
$XDG_CACHE_HOME/furst-read and opened in $FURST_BROWSER, $BROWSER, or the
|
||||
first light browser found on $PATH.
|
||||
";
|
||||
|
||||
const BROWSERS: &[&str] = &["surf", "luakit", "vimb", "badwolf", "epiphany", "firefox"];
|
||||
const LIMIT: usize = 8 * 1024 * 1024;
|
||||
|
||||
enum Sink {
|
||||
Stdout(bool),
|
||||
File(PathBuf),
|
||||
Open,
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
match run() {
|
||||
Ok(()) => ExitCode::SUCCESS,
|
||||
Err(e) => {
|
||||
eprintln!("furst-read: {e}");
|
||||
ExitCode::FAILURE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn run() -> Result<(), String> {
|
||||
let args: Vec<String> = env::args().skip(1).collect();
|
||||
if args.is_empty() || args.iter().any(|a| a == "-h" || a == "--help") {
|
||||
print!("{USAGE}");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut sink = Sink::Open;
|
||||
let mut images = true;
|
||||
let mut stdin = false;
|
||||
let mut browser: Option<String> = None;
|
||||
let mut url: Option<String> = None;
|
||||
|
||||
let mut it = args.into_iter();
|
||||
while let Some(arg) = it.next() {
|
||||
match arg.as_str() {
|
||||
"--html" => sink = Sink::Stdout(false),
|
||||
"--text" => sink = Sink::Stdout(true),
|
||||
"--open" => sink = Sink::Open,
|
||||
"--no-images" => images = false,
|
||||
"--stdin" => stdin = true,
|
||||
"--out" => sink = Sink::File(it.next().ok_or("--out needs a path")?.into()),
|
||||
"--browser" => browser = Some(it.next().ok_or("--browser needs a command")?),
|
||||
a if a.starts_with('-') => return Err(format!("unknown flag {a} (--help for usage)")),
|
||||
a => url = Some(a.to_string()),
|
||||
}
|
||||
}
|
||||
let url = url.ok_or("no url given")?;
|
||||
|
||||
let page = if stdin {
|
||||
let mut buf = Vec::new();
|
||||
io::stdin().read_to_end(&mut buf).map_err(|e| e.to_string())?;
|
||||
fetch::Page { url: url.clone(), html: fetch::decode(&buf, "") }
|
||||
} else {
|
||||
fetch::get(&url, LIMIT)?
|
||||
};
|
||||
|
||||
let mut page = page;
|
||||
let mut doc = scraper::Html::parse_document(&page.html);
|
||||
// Redirect stubs: a page whose only content is "go here instead".
|
||||
if !stdin {
|
||||
for _ in 0..3 {
|
||||
let Some(next) = extract::meta_refresh(&doc, &page.url) else { break };
|
||||
if next == page.url {
|
||||
break;
|
||||
}
|
||||
page = fetch::get(&next, LIMIT)?;
|
||||
doc = scraper::Html::parse_document(&page.html);
|
||||
}
|
||||
}
|
||||
let article = extract::extract(&doc);
|
||||
|
||||
match sink {
|
||||
Sink::Stdout(as_text) => {
|
||||
let body = if as_text {
|
||||
render::text(&doc, &article)
|
||||
} else {
|
||||
render::html(&doc, &article, &page.url, &render::Options { images })
|
||||
};
|
||||
// A closed pipe is not an error: `furst-read --text url | head`.
|
||||
if let Err(e) = io::stdout().write_all(body.as_bytes())
|
||||
&& e.kind() != io::ErrorKind::BrokenPipe
|
||||
{
|
||||
return Err(e.to_string());
|
||||
}
|
||||
}
|
||||
Sink::File(path) => {
|
||||
let body = render::html(&doc, &article, &page.url, &render::Options { images });
|
||||
fs::write(&path, body).map_err(|e| format!("{}: {e}", path.display()))?;
|
||||
println!("{}", path.display());
|
||||
}
|
||||
Sink::Open => {
|
||||
let body = render::html(&doc, &article, &page.url, &render::Options { images });
|
||||
let path = cache_path(&page.url)?;
|
||||
fs::write(&path, body).map_err(|e| format!("{}: {e}", path.display()))?;
|
||||
let cmd = browser
|
||||
.or_else(|| env::var("FURST_BROWSER").ok().filter(|s| !s.is_empty()))
|
||||
.or_else(|| env::var("BROWSER").ok().filter(|s| !s.is_empty()))
|
||||
.or_else(|| BROWSERS.iter().find(|b| which(b)).map(|b| b.to_string()))
|
||||
.ok_or("no browser found; set $FURST_BROWSER or use --out")?;
|
||||
let target = format!("file://{}", path.display());
|
||||
let err = Command::new(&cmd).arg(&target).exec();
|
||||
return Err(format!("{cmd}: {err}"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Named by a hash of the URL, so revisiting a page reuses the file rather
|
||||
/// than littering, and so two open articles do not collide.
|
||||
fn cache_path(url: &str) -> Result<PathBuf, String> {
|
||||
let dir = env::var_os("XDG_CACHE_HOME")
|
||||
.map(PathBuf::from)
|
||||
.filter(|p| !p.as_os_str().is_empty())
|
||||
.or_else(|| env::var_os("HOME").map(|h| PathBuf::from(h).join(".cache")))
|
||||
.ok_or("neither $XDG_CACHE_HOME nor $HOME is set")?
|
||||
.join("furst-read");
|
||||
fs::create_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?;
|
||||
Ok(dir.join(format!("{:016x}.html", fnv1a(url))))
|
||||
}
|
||||
|
||||
fn fnv1a(s: &str) -> u64 {
|
||||
let mut h: u64 = 0xcbf2_9ce4_8422_2325;
|
||||
for b in s.bytes() {
|
||||
h ^= b as u64;
|
||||
h = h.wrapping_mul(0x1000_0000_01b3);
|
||||
}
|
||||
h
|
||||
}
|
||||
|
||||
fn which(cmd: &str) -> bool {
|
||||
env::var_os("PATH").is_some_and(|p| env::split_paths(&p).any(|d| d.join(cmd).is_file()))
|
||||
}
|
||||
352
furst-read/src/render.rs
Normal file
352
furst-read/src/render.rs
Normal file
@ -0,0 +1,352 @@
|
||||
//! Serialise the extracted nodes as a small, self-contained page: no scripts,
|
||||
//! no webfonts, no third-party requests. On a Core2 the cost of a page is
|
||||
//! dominated by what it asks the machine to do, not by how it looks.
|
||||
|
||||
use std::fmt::Write as _;
|
||||
|
||||
use ego_tree::NodeRef;
|
||||
use scraper::node::Element;
|
||||
use scraper::{Html, Node};
|
||||
|
||||
use crate::extract::Article;
|
||||
use crate::urljoin::join;
|
||||
|
||||
/// Rendered verbatim, with attributes filtered.
|
||||
const KEEP: &[&str] = &[
|
||||
"p", "h1", "h2", "h3", "h4", "h5", "h6", "ul", "ol", "li", "blockquote", "pre", "code", "em",
|
||||
"strong", "b", "i", "a", "figure", "figcaption", "table", "thead", "tbody", "tfoot", "tr", "th",
|
||||
"td", "caption", "sup", "sub", "dl", "dt", "dd", "cite", "q", "s", "u", "mark", "small", "abbr",
|
||||
];
|
||||
/// Dropped with their entire subtree.
|
||||
const DROP: &[&str] = &[
|
||||
"script", "style", "noscript", "iframe", "svg", "canvas", "form", "input", "button", "select",
|
||||
"textarea", "nav", "aside", "object", "embed", "link", "meta", "template", "video", "audio",
|
||||
];
|
||||
const VOID: &[&str] = &["br", "hr"];
|
||||
|
||||
pub struct Options {
|
||||
pub images: bool,
|
||||
}
|
||||
|
||||
struct Cx<'a> {
|
||||
base: &'a str,
|
||||
images: bool,
|
||||
}
|
||||
|
||||
pub fn html(doc: &Html, article: &Article, base: &str, opts: &Options) -> String {
|
||||
let cx = Cx { base, images: opts.images };
|
||||
let mut body = String::new();
|
||||
for id in &article.nodes {
|
||||
if let Some(node) = doc.tree.get(*id) {
|
||||
body.push_str(&node_html(node, &cx, false));
|
||||
}
|
||||
}
|
||||
|
||||
let mut meta = Vec::new();
|
||||
if let Some(b) = &article.byline {
|
||||
meta.push(escape_text(b));
|
||||
}
|
||||
if let Some(s) = &article.site {
|
||||
meta.push(escape_text(s));
|
||||
}
|
||||
meta.push(format!("<a href=\"{}\">original</a>", escape_attr(base)));
|
||||
|
||||
let title = if article.title.is_empty() { base } else { &article.title };
|
||||
format!(
|
||||
"<!doctype html>\n<html lang=\"\"><head>\n<meta charset=\"utf-8\">\n\
|
||||
<meta name=\"viewport\" content=\"width=device-width,initial-scale=1\">\n\
|
||||
<title>{}</title>\n<style>{CSS}</style>\n</head>\n<body>\n<main>\n\
|
||||
<h1>{}</h1>\n<p class=\"meta\">{}</p>\n{}</main>\n</body></html>\n",
|
||||
escape_text(title),
|
||||
escape_text(&article.title),
|
||||
meta.join(" · "),
|
||||
body
|
||||
)
|
||||
}
|
||||
|
||||
pub fn text(doc: &Html, article: &Article) -> String {
|
||||
let mut out = String::new();
|
||||
if !article.title.is_empty() {
|
||||
let _ = writeln!(out, "{}\n", article.title);
|
||||
}
|
||||
for id in &article.nodes {
|
||||
if let Some(node) = doc.tree.get(*id) {
|
||||
node_text(node, &mut out, false);
|
||||
}
|
||||
}
|
||||
// Never more than one blank line in a row.
|
||||
let mut clean = String::with_capacity(out.len());
|
||||
let mut blanks = 0;
|
||||
for line in out.lines() {
|
||||
let line = line.trim_end();
|
||||
if line.is_empty() {
|
||||
blanks += 1;
|
||||
if blanks > 1 {
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
blanks = 0;
|
||||
}
|
||||
clean.push_str(line);
|
||||
clean.push('\n');
|
||||
}
|
||||
clean
|
||||
}
|
||||
|
||||
fn node_html(node: NodeRef<Node>, cx: &Cx, in_pre: bool) -> String {
|
||||
match node.value() {
|
||||
Node::Text(t) => escape_text(&if in_pre { t.text.to_string() } else { collapse(&t.text) }),
|
||||
Node::Element(e) => element_html(node, e, cx, in_pre),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn element_html(node: NodeRef<Node>, e: &Element, cx: &Cx, in_pre: bool) -> String {
|
||||
let name = e.name();
|
||||
if DROP.contains(&name) {
|
||||
return String::new();
|
||||
}
|
||||
if name == "img" || name == "picture" {
|
||||
return if cx.images { img_html(node, e, cx) } else { String::new() };
|
||||
}
|
||||
if VOID.contains(&name) {
|
||||
return format!("<{name}>");
|
||||
}
|
||||
|
||||
let pre = in_pre || name == "pre";
|
||||
let inner: String = node.children().map(|c| node_html(c, cx, pre)).collect();
|
||||
|
||||
if name == "a" {
|
||||
let href = e.attr("href").unwrap_or("").trim();
|
||||
// An anchor to nothing, to script, or to an id we did not keep is just text.
|
||||
if href.is_empty() || href.starts_with('#') || href.to_ascii_lowercase().starts_with("javascript:") {
|
||||
return inner;
|
||||
}
|
||||
if inner.trim().is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
return format!("<a href=\"{}\">{inner}</a>", escape_attr(&join(cx.base, href)));
|
||||
}
|
||||
|
||||
// Anything not on the list contributes its children but no tag of its own,
|
||||
// which is how the wrapper divs disappear.
|
||||
if !KEEP.contains(&name) {
|
||||
return inner;
|
||||
}
|
||||
if inner.trim().is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
format!("<{name}{}>{inner}</{name}>", attrs_html(e, name))
|
||||
}
|
||||
|
||||
fn attrs_html(e: &Element, name: &str) -> String {
|
||||
let mut out = String::new();
|
||||
if matches!(name, "td" | "th") {
|
||||
for key in ["colspan", "rowspan"] {
|
||||
if let Some(v) = e.attr(key)
|
||||
&& v.chars().all(|c| c.is_ascii_digit())
|
||||
&& !v.is_empty()
|
||||
{
|
||||
let _ = write!(out, " {key}=\"{v}\"");
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn img_html(node: NodeRef<Node>, e: &Element, cx: &Cx) -> String {
|
||||
// <picture> holds the <img>; find it.
|
||||
if e.name() == "picture" {
|
||||
return node.children().map(|c| node_html(c, cx, false)).collect();
|
||||
}
|
||||
let Some(src) = img_src(e) else { return String::new() };
|
||||
let alt = e.attr("alt").map(escape_attr).unwrap_or_default();
|
||||
format!("<img src=\"{}\" alt=\"{alt}\" loading=\"lazy\">", escape_attr(&join(cx.base, &src)))
|
||||
}
|
||||
|
||||
fn img_src(e: &Element) -> Option<String> {
|
||||
let usable = |s: &str| {
|
||||
let s = s.trim();
|
||||
!s.is_empty() && !s.starts_with("data:image/gif") && !s.starts_with("data:image/svg")
|
||||
};
|
||||
for key in ["src", "data-src", "data-original", "data-lazy-src", "data-actualsrc"] {
|
||||
if let Some(v) = e.attr(key)
|
||||
&& usable(v)
|
||||
{
|
||||
return Some(v.trim().to_string());
|
||||
}
|
||||
}
|
||||
// First srcset candidate, not the last: on this hardware the smallest
|
||||
// variant is the one we want.
|
||||
let set = e.attr("srcset").or_else(|| e.attr("data-srcset"))?;
|
||||
let first = set.split(',').next()?.split_whitespace().next()?;
|
||||
usable(first).then(|| first.to_string())
|
||||
}
|
||||
|
||||
fn node_text(node: NodeRef<Node>, out: &mut String, in_pre: bool) {
|
||||
match node.value() {
|
||||
Node::Text(t) => out.push_str(&if in_pre { t.text.to_string() } else { collapse(&t.text) }),
|
||||
Node::Element(e) => {
|
||||
let name = e.name();
|
||||
if DROP.contains(&name) || name == "img" {
|
||||
return;
|
||||
}
|
||||
let block = matches!(
|
||||
name,
|
||||
"p" | "div" | "h1" | "h2" | "h3" | "h4" | "h5" | "h6" | "blockquote" | "pre" | "tr" | "figcaption" | "section"
|
||||
);
|
||||
if name == "li" {
|
||||
out.push_str("\n - ");
|
||||
} else if block {
|
||||
out.push_str("\n\n");
|
||||
} else if name == "br" {
|
||||
out.push('\n');
|
||||
}
|
||||
let pre = in_pre || name == "pre";
|
||||
for child in node.children() {
|
||||
node_text(child, out, pre);
|
||||
}
|
||||
if block || name == "li" {
|
||||
out.push('\n');
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// Collapse whitespace runs without trimming: the spaces between inline tags
|
||||
/// are load-bearing.
|
||||
fn collapse(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
let mut in_ws = false;
|
||||
for c in s.chars() {
|
||||
if c.is_whitespace() {
|
||||
if !in_ws {
|
||||
out.push(' ');
|
||||
in_ws = true;
|
||||
}
|
||||
} else {
|
||||
out.push(c);
|
||||
in_ws = false;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn escape_text(s: &str) -> String {
|
||||
s.replace('&', "&").replace('<', "<").replace('>', ">")
|
||||
}
|
||||
|
||||
fn escape_attr(s: &str) -> String {
|
||||
escape_text(s).replace('"', """)
|
||||
}
|
||||
|
||||
const CSS: &str = "\
|
||||
:root{color-scheme:light dark}\
|
||||
*{box-sizing:border-box}\
|
||||
body{margin:0;padding:2.5rem 1.25rem 6rem;color:#1a1a1a;background:#fdfdfc;\
|
||||
font:16px/1.65 -apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,'Helvetica Neue',Arial,sans-serif}\
|
||||
main{max-width:38rem;margin:0 auto}\
|
||||
h1{font-size:1.8rem;line-height:1.25;margin:0 0 .6rem}\
|
||||
.meta{color:#6b6b6b;font-size:.875rem;margin:0 0 2.5rem;padding-bottom:1.25rem;border-bottom:1px solid #e3e3e0}\
|
||||
p{margin:0 0 1.25rem}\
|
||||
h2,h3,h4,h5,h6{line-height:1.3;margin:2.25rem 0 .75rem}\
|
||||
h2{font-size:1.3rem}h3{font-size:1.1rem}h4,h5,h6{font-size:1rem}\
|
||||
a{color:#0b57a4}\
|
||||
img{max-width:100%;height:auto;display:block;margin:1.5rem auto}\
|
||||
figure{margin:1.5rem 0}\
|
||||
figcaption{font-size:.85rem;color:#6b6b6b;text-align:center;margin-top:.5rem}\
|
||||
blockquote{margin:1.5rem 0;padding-left:1rem;border-left:3px solid #d8d8d4;color:#4a4a4a}\
|
||||
pre{overflow-x:auto;padding:.9rem 1rem;background:#f4f4f1;border-radius:4px;font-size:.875rem;line-height:1.45}\
|
||||
code{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:.9em}\
|
||||
p code,li code{background:#f0f0ec;padding:.1em .3em;border-radius:3px}\
|
||||
pre code{background:none;padding:0}\
|
||||
ul,ol{margin:0 0 1.25rem;padding-left:1.5rem}li{margin:.35rem 0}\
|
||||
table{border-collapse:collapse;width:100%;margin:1.5rem 0;font-size:.9rem;display:block;overflow-x:auto}\
|
||||
th,td{border:1px solid #ddd;padding:.4rem .6rem;text-align:left}\
|
||||
hr{border:0;border-top:1px solid #e3e3e0;margin:2rem 0}\
|
||||
@media(prefers-color-scheme:dark){\
|
||||
body{color:#d8d6d1;background:#141414}\
|
||||
.meta{color:#8e8b85;border-color:#2b2b2b}\
|
||||
a{color:#7fb3e8}\
|
||||
blockquote{border-color:#3a3a3a;color:#a9a6a0}\
|
||||
pre{background:#1d1d1d}p code,li code{background:#242424}\
|
||||
th,td{border-color:#333}\
|
||||
hr{border-color:#2b2b2b}}";
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::extract;
|
||||
|
||||
fn render(body: &str) -> String {
|
||||
let doc = Html::parse_document(&format!("<html><body>{body}</body></html>"));
|
||||
let root = doc.select(&scraper::Selector::parse("body").unwrap()).next().unwrap().id();
|
||||
let article = extract::Article {
|
||||
title: "T".into(),
|
||||
byline: None,
|
||||
site: None,
|
||||
nodes: vec![root],
|
||||
};
|
||||
html(&doc, &article, "https://ex.test/a/b.html", &Options { images: true })
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn links_and_images_become_absolute() {
|
||||
let out = render(r#"<p><a href="../c.html">x</a></p><img src="i.png" alt="A">"#);
|
||||
assert!(out.contains(r#"<a href="https://ex.test/c.html">x</a>"#));
|
||||
assert!(out.contains(r#"<img src="https://ex.test/a/i.png" alt="A""#));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lazy_images_use_the_real_source() {
|
||||
let out = render(r#"<img src="data:image/gif;base64,R0lGOD" data-src="/real.jpg">"#);
|
||||
assert!(out.contains("https://ex.test/real.jpg"));
|
||||
assert!(!out.contains("data:image/gif"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn srcset_takes_the_smallest_candidate() {
|
||||
let out = render(r#"<img srcset="/s.jpg 320w, /l.jpg 1600w">"#);
|
||||
assert!(out.contains("/s.jpg"));
|
||||
assert!(!out.contains("/l.jpg"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scripts_styles_and_wrappers_are_stripped() {
|
||||
let out = render(r#"<div class="x"><script>evil()</script><style>a{}</style><p>keep</p></div>"#);
|
||||
assert!(out.contains("<p>keep</p>"));
|
||||
assert!(!out.contains("evil"));
|
||||
assert!(!out.contains("<div"));
|
||||
assert!(!out.contains("class=\"x\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dead_anchors_are_unwrapped_and_empty_tags_dropped() {
|
||||
let out = render(r##"<p><a href="javascript:x()">t</a></p><p></p><p><a href="#f">g</a></p>"##);
|
||||
assert!(out.contains("<p>t</p>"));
|
||||
assert!(out.contains("<p>g</p>"));
|
||||
assert!(!out.contains("javascript:"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_is_escaped() {
|
||||
let out = render(r#"<p>a < b & c</p>"#);
|
||||
assert!(out.contains("a < b & c"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_keeps_its_whitespace_but_prose_does_not() {
|
||||
let out = render("<pre>a\n b</pre><p>x \n y</p>");
|
||||
assert!(out.contains("<pre>a\n b</pre>"));
|
||||
assert!(out.contains("<p>x y</p>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_page_pulls_in_nothing_external() {
|
||||
let out = render("<p>hi</p>");
|
||||
assert!(!out.contains("<script"));
|
||||
assert!(!out.contains("googleapis"));
|
||||
assert!(!out.contains("@import"));
|
||||
}
|
||||
}
|
||||
137
furst-read/src/urljoin.rs
Normal file
137
furst-read/src/urljoin.rs
Normal file
@ -0,0 +1,137 @@
|
||||
//! Relative reference resolution, RFC 3986 §5.3 in the shape this tool needs.
|
||||
//! Extracted links must be absolute, or the reader's output only works while
|
||||
//! it happens to sit at the right path.
|
||||
|
||||
struct Parts<'a> {
|
||||
scheme: &'a str,
|
||||
authority: &'a str,
|
||||
path: &'a str,
|
||||
query: &'a str,
|
||||
}
|
||||
|
||||
pub fn join(base: &str, href: &str) -> String {
|
||||
let href = href.trim();
|
||||
if href.is_empty() {
|
||||
return base.to_string();
|
||||
}
|
||||
if has_scheme(href) {
|
||||
return href.to_string();
|
||||
}
|
||||
let b = split(base);
|
||||
// Protocol-relative: //cdn.example/x.png
|
||||
if let Some(rest) = href.strip_prefix("//") {
|
||||
return format!("{}://{rest}", b.scheme);
|
||||
}
|
||||
let origin = format!("{}://{}", b.scheme, b.authority);
|
||||
if let Some(frag) = href.strip_prefix('#') {
|
||||
return format!("{origin}{}{}#{frag}", b.path, b.query);
|
||||
}
|
||||
if href.starts_with('?') {
|
||||
return format!("{origin}{}{href}", b.path);
|
||||
}
|
||||
|
||||
let cut = href.find(['?', '#']).unwrap_or(href.len());
|
||||
let (hpath, suffix) = href.split_at(cut);
|
||||
let merged = if hpath.starts_with('/') {
|
||||
hpath.to_string()
|
||||
} else {
|
||||
// Everything up to and including the base's last slash.
|
||||
let dir = match b.path.rfind('/') {
|
||||
Some(i) => &b.path[..=i],
|
||||
None => "/",
|
||||
};
|
||||
format!("{dir}{hpath}")
|
||||
};
|
||||
format!("{origin}{}{suffix}", remove_dot_segments(&merged))
|
||||
}
|
||||
|
||||
fn split(u: &str) -> Parts<'_> {
|
||||
let (scheme, rest) = match u.find("://") {
|
||||
Some(i) if has_scheme(&u[..i + 1]) => (&u[..i], &u[i + 3..]),
|
||||
_ => ("https", u),
|
||||
};
|
||||
let rest = rest.split('#').next().unwrap_or("");
|
||||
let (rest, query) = match rest.find('?') {
|
||||
Some(i) => (&rest[..i], &rest[i..]),
|
||||
None => (rest, ""),
|
||||
};
|
||||
let (authority, path) = match rest.find('/') {
|
||||
Some(i) => (&rest[..i], &rest[i..]),
|
||||
None => (rest, "/"),
|
||||
};
|
||||
Parts { scheme, authority, path, query }
|
||||
}
|
||||
|
||||
fn has_scheme(s: &str) -> bool {
|
||||
match s.find(':') {
|
||||
Some(i) => {
|
||||
let head = &s[..i];
|
||||
!head.is_empty()
|
||||
&& !head.contains('/')
|
||||
&& head.starts_with(|c: char| c.is_ascii_alphabetic())
|
||||
&& head.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
|
||||
}
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_dot_segments(path: &str) -> String {
|
||||
let keep_trailing =
|
||||
path.ends_with('/') || path.ends_with("/.") || path.ends_with("/..") || path == ".." || path == ".";
|
||||
let mut out: Vec<&str> = Vec::new();
|
||||
for seg in path.split('/') {
|
||||
match seg {
|
||||
"" | "." => {}
|
||||
".." => {
|
||||
out.pop();
|
||||
}
|
||||
s => out.push(s),
|
||||
}
|
||||
}
|
||||
let mut s = format!("/{}", out.join("/"));
|
||||
if keep_trailing && !s.ends_with('/') {
|
||||
s.push('/');
|
||||
}
|
||||
s
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::join;
|
||||
|
||||
const B: &str = "https://ex.test/a/b/page.html?x=1";
|
||||
|
||||
#[test]
|
||||
fn absolute_and_protocol_relative_pass_through() {
|
||||
assert_eq!(join(B, "http://other.test/z"), "http://other.test/z");
|
||||
assert_eq!(join(B, "//cdn.test/i.png"), "https://cdn.test/i.png");
|
||||
assert_eq!(join(B, "mailto:a@b.test"), "mailto:a@b.test");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relative_forms() {
|
||||
assert_eq!(join(B, "c.html"), "https://ex.test/a/b/c.html");
|
||||
assert_eq!(join(B, "/c.html"), "https://ex.test/c.html");
|
||||
assert_eq!(join(B, "../c.html"), "https://ex.test/a/c.html");
|
||||
assert_eq!(join(B, "../../c.html"), "https://ex.test/c.html");
|
||||
assert_eq!(join(B, "./c.html"), "https://ex.test/a/b/c.html");
|
||||
assert_eq!(join(B, "sub/"), "https://ex.test/a/b/sub/");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_and_fragment_keep_the_base_path() {
|
||||
assert_eq!(join(B, "#sec"), "https://ex.test/a/b/page.html?x=1#sec");
|
||||
assert_eq!(join(B, "?y=2"), "https://ex.test/a/b/page.html?y=2");
|
||||
assert_eq!(join(B, "c.html?y=2#s"), "https://ex.test/a/b/c.html?y=2#s");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cannot_escape_above_the_root() {
|
||||
assert_eq!(join(B, "../../../../etc"), "https://ex.test/etc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bare_origin_base() {
|
||||
assert_eq!(join("https://ex.test", "c.html"), "https://ex.test/c.html");
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user