From 3f71d324c9c31f5b65c3416aa3571dd9e9d3d327 Mon Sep 17 00:00:00 2001 From: nak0x Date: Sun, 6 Sep 2026 20:07:49 +0200 Subject: [PATCH] Add search, feeds and listing views to the reader server Extraction looks for prose, so a front page, a comment thread or a search result page correctly yields almost nothing. These render the structure instead, and every link they emit routes back through /read. /search?q= results from a configurable HTML endpoint /feed?u= RSS and Atom /read?u= now picks a site view, falling back to a link index when a page has too little text to be an article Feeds are scanned rather than parsed. A feed needs five fields per entry and an HTML parser mangles XML, so this walks the tags directly: CDATA, named and numeric entities, and Atom's preference for rel=alternate over rel=self. No XML dependency. Hacker News gets a real adapter: stories with score, author and a link into the discussion, and comment threads rendered with their indent preserved. Comment bodies go through the article renderer so links inside them behave like every other link. Search is deliberately engine-agnostic. Known result shapes are tried first, then heading links, then any link, because every free HTML endpoint eventually rate-limits a repeat visitor. When one answers with a challenge page rather than results the reader says so and points at the config, instead of showing an empty page; detection reads the body, since these arrive as 200 or 202 rather than an error status. A blocked search is never cached. Pages that turn out to be feeds redirect to the feed view, and a page that declares its own feed offers it. --- furst-read/src/fetch.rs | 4 +- furst-read/src/main.rs | 2 +- furst-serve/src/config.rs | 22 ++++ furst-serve/src/feed.rs | 238 +++++++++++++++++++++++++++++++++++++ furst-serve/src/handler.rs | 192 +++++++++++++++++++++++++----- furst-serve/src/main.rs | 3 + furst-serve/src/search.rs | 238 +++++++++++++++++++++++++++++++++++++ furst-serve/src/sites.rs | 207 ++++++++++++++++++++++++++++++++ 8 files changed, 876 insertions(+), 30 deletions(-) create mode 100644 furst-serve/src/feed.rs create mode 100644 furst-serve/src/search.rs create mode 100644 furst-serve/src/sites.rs diff --git a/furst-read/src/fetch.rs b/furst-read/src/fetch.rs index 40cd0b9..9bb3284 100644 --- a/furst-read/src/fetch.rs +++ b/furst-read/src/fetch.rs @@ -11,6 +11,7 @@ pub struct Page { pub url: String, pub html: String, pub content_type: String, + pub status: u16, } pub fn get(url: &str, limit: usize) -> Result { @@ -22,6 +23,7 @@ pub fn get(url: &str, limit: usize) -> Result { .into(); let mut resp = agent.get(url).call().map_err(|e| format!("{url}: {e}"))?; + let status = resp.status().as_u16(); let final_url = resp.get_uri().to_string(); let content_type = resp .headers() @@ -37,7 +39,7 @@ pub fn get(url: &str, limit: usize) -> Result { .read_to_vec() .map_err(|e| format!("{url}: {e}"))?; - Ok(Page { url: final_url, html: decode(&bytes, &content_type), content_type }) + Ok(Page { url: final_url, html: decode(&bytes, &content_type), content_type, status }) } /// Decode to text using, in order: the Content-Type charset, a `` diff --git a/furst-read/src/main.rs b/furst-read/src/main.rs index 54fc99d..f3daf02 100644 --- a/furst-read/src/main.rs +++ b/furst-read/src/main.rs @@ -78,7 +78,7 @@ fn run() -> Result<(), String> { let page = if stdin { let mut buf = Vec::new(); io::stdin().read_to_end(&mut buf).map_err(|e| e.to_string())?; - fetch::Page { url: url.clone(), html: fetch::decode(&buf, ""), content_type: String::new() } + fetch::Page { url: url.clone(), html: fetch::decode(&buf, ""), content_type: String::new(), status: 200 } } else { fetch::get(&url, LIMIT)? }; diff --git a/furst-serve/src/config.rs b/furst-serve/src/config.rs index 3479dc9..bbf5d03 100644 --- a/furst-serve/src/config.rs +++ b/furst-serve/src/config.rs @@ -12,6 +12,21 @@ pub struct Home { pub links: Vec, #[serde(default, rename = "feed")] pub feeds: Vec, + #[serde(default)] + pub search: Search, +} + +#[derive(Debug, Deserialize)] +pub struct Search { + /// Result-page URL with {q} where the encoded query goes. Any engine that + /// serves plain HTML works; the parser falls back to reading links. + pub url: String, +} + +impl Default for Search { + fn default() -> Self { + Search { url: "https://html.duckduckgo.com/html/?q={q}".to_string() } + } } #[derive(Debug, Deserialize)] @@ -63,4 +78,11 @@ url = "https://lwn.net/headlines/newrss" [[feed]] name = "Phoronix" url = "https://www.phoronix.com/rss.php" + +# Any engine that serves plain HTML. {q} is the encoded query. +# DuckDuckGo rate-limits repeat visitors; a SearXNG instance does not. +[search] +url = "https://html.duckduckgo.com/html/?q={q}" +# url = "https://searx.be/search?q={q}" +# url = "https://search.marginalia.nu/search?query={q}" "#; diff --git a/furst-serve/src/feed.rs b/furst-serve/src/feed.rs new file mode 100644 index 0000000..860d289 --- /dev/null +++ b/furst-serve/src/feed.rs @@ -0,0 +1,238 @@ +//! RSS and Atom, scanned rather than parsed. +//! +//! A feed only ever needs five fields per entry, and an HTML parser mangles +//! XML, so this walks the tags directly. No XML dependency, no surprises. + +pub struct Entry { + pub title: String, + pub link: String, + pub date: String, + pub summary: String, +} + +pub struct Feed { + pub title: String, + pub entries: Vec, +} + +pub fn looks_like_feed(content_type: &str, body: &str) -> bool { + let ct = content_type.to_ascii_lowercase(); + if ct.contains("rss") || ct.contains("atom") || ct.contains("xml") { + return true; + } + let head = &body[..body.len().min(1024)].to_ascii_lowercase(); + head.contains(" Option { + let mut entries = Vec::new(); + let mut cursor = 0; + let mut first_item = xml.len(); + + while entries.len() < 200 { + let item = ["item", "entry"] + .iter() + .filter_map(|name| block(xml, name, cursor)) + .min_by_key(|(start, _, _)| *start); + let Some((start, end, block)) = item else { break }; + first_item = first_item.min(start); + cursor = end; + + let title = tag_text(block, "title").unwrap_or_default(); + let link = entry_link(block).unwrap_or_default(); + if title.is_empty() && link.is_empty() { + continue; + } + let date = ["pubDate", "published", "updated", "dc:date"] + .iter() + .find_map(|n| tag_text(block, n)) + .unwrap_or_default(); + let summary = ["description", "summary", "content"] + .iter() + .find_map(|n| tag_text(block, n)) + .map(|s| strip_tags(&s)) + .unwrap_or_default(); + + entries.push(Entry { title, link, date, summary }); + } + + if entries.is_empty() { + return None; + } + // The channel title is whichever comes before the first entry. + let title = tag_text(&xml[..first_item], "title").unwrap_or_default(); + Some(Feed { title, entries }) +} + +/// Atom puts the URL in an attribute and may list several; RSS uses text. +fn entry_link(block: &str) -> Option<String> { + if let Some(text) = tag_text(block, "link").filter(|s| !s.is_empty()) { + return Some(text); + } + let mut best: Option<String> = None; + let mut cursor = 0; + while let Some(i) = block[cursor..].find("<link") { + let start = cursor + i; + let end = block[start..].find('>').map(|e| start + e)?; + let tag = &block[start..end]; + cursor = end; + let Some(href) = attr(tag, "href") else { continue }; + let rel = attr(tag, "rel").unwrap_or_default(); + if rel.is_empty() || rel == "alternate" { + return Some(href); + } + best.get_or_insert(href); + } + best +} + +/// Locate `<name ...> ... </name>`, returning its bounds and inner text. +fn block<'a>(xml: &'a str, name: &str, from: usize) -> Option<(usize, usize, &'a str)> { + let open = format!("<{name}"); + let close = format!("</{name}>"); + let mut cursor = from; + loop { + let start = cursor + xml[cursor..].find(&open)?; + // `<item` must not match `<items`. + let after = xml[start + open.len()..].chars().next(); + if after.is_some_and(|c| c.is_ascii_alphanumeric() || c == '-' || c == ':') { + cursor = start + open.len(); + continue; + } + let body_start = start + xml[start..].find('>')? + 1; + let end = body_start + xml[body_start..].find(&close)?; + return Some((start, end + close.len(), &xml[body_start..end])); + } +} + +fn tag_text(block: &str, name: &str) -> Option<String> { + let (_, _, inner) = self::block(block, name, 0)?; + Some(decode_entities(strip_cdata(inner).trim())) +} + +fn attr(tag: &str, name: &str) -> Option<String> { + let key = format!("{name}="); + let i = tag.find(&key)? + key.len(); + let rest = &tag[i..]; + let quote = rest.chars().next()?; + if quote == '"' || quote == '\'' { + let end = rest[1..].find(quote)? + 1; + Some(decode_entities(&rest[1..end])) + } else { + let end = rest.find([' ', '>', '/']).unwrap_or(rest.len()); + Some(decode_entities(&rest[..end])) + } +} + +fn strip_cdata(s: &str) -> &str { + s.trim().strip_prefix("<![CDATA[").and_then(|r| r.strip_suffix("]]>")).unwrap_or(s) +} + +pub fn strip_tags(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut in_tag = false; + for c in s.chars() { + match c { + '<' => in_tag = true, + '>' => in_tag = false, + _ if !in_tag => out.push(c), + _ => {} + } + } + decode_entities(&out).split_whitespace().collect::<Vec<_>>().join(" ") +} + +fn decode_entities(s: &str) -> String { + if !s.contains('&') { + return s.to_string(); + } + let mut out = String::with_capacity(s.len()); + let mut rest = s; + while let Some(i) = rest.find('&') { + out.push_str(&rest[..i]); + let tail = &rest[i..]; + let Some(semi) = tail[..tail.len().min(12)].find(';') else { + out.push('&'); + rest = &tail[1..]; + continue; + }; + let name = &tail[1..semi]; + let decoded = match name { + "amp" => Some('&'), + "lt" => Some('<'), + "gt" => Some('>'), + "quot" => Some('"'), + "apos" => Some('\''), + "nbsp" => Some(' '), + n if n.starts_with("#x") || n.starts_with("#X") => { + u32::from_str_radix(&n[2..], 16).ok().and_then(char::from_u32) + } + n if n.starts_with('#') => n[1..].parse().ok().and_then(char::from_u32), + _ => None, + }; + match decoded { + Some(c) => out.push(c), + None => out.push_str(&tail[..=semi]), + } + rest = &tail[semi + 1..]; + } + out.push_str(rest); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + const RSS: &str = r#"<?xml version="1.0"?><rss><channel> + <title>My Feedhttps://ex.test/ + First & besthttps://ex.test/1 + Mon, 01 Jan 2026 + Hello there

]]> + Secondhttps://ex.test/2 + "#; + + const ATOM: &str = r#"Atom Feed + + Entry one + + + 2026-01-01A summary"#; + + #[test] + fn parses_rss_with_cdata_and_entities() { + let f = parse(RSS).unwrap(); + assert_eq!(f.title, "My Feed"); + assert_eq!(f.entries.len(), 2); + assert_eq!(f.entries[0].title, "First & best"); + assert_eq!(f.entries[0].link, "https://ex.test/1"); + assert_eq!(f.entries[0].summary, "Hello there"); + assert_eq!(f.entries[1].title, "Second"); + } + + #[test] + fn atom_prefers_the_alternate_link_over_self() { + let f = parse(ATOM).unwrap(); + assert_eq!(f.title, "Atom Feed"); + assert_eq!(f.entries[0].link, "https://ex.test/one"); + assert_eq!(f.entries[0].summary, "A summary"); + } + + #[test] + fn detects_feeds_by_type_or_shape() { + assert!(looks_like_feed("application/rss+xml", "")); + assert!(looks_like_feed("", "")); + assert!(!looks_like_feed("text/html", "")); + } + + #[test] + fn a_document_with_no_items_is_not_a_feed() { + assert!(parse("

hi

").is_none()); + } + + #[test] + fn numeric_entities_decode() { + assert_eq!(decode_entities("a'b — c"), "a'b — c"); + assert_eq!(decode_entities("100% ¬anentity"), "100% ¬anentity"); + } +} diff --git a/furst-serve/src/handler.rs b/furst-serve/src/handler.rs index 01f67ca..79a5fb7 100644 --- a/furst-serve/src/handler.rs +++ b/furst-serve/src/handler.rs @@ -10,7 +10,7 @@ use scraper::Html; use crate::cache::Cache; use crate::config::Home; use crate::http::{Request, Response}; -use crate::page; +use crate::{feed, page, search, sites}; const LIMIT: usize = 8 * 1024 * 1024; /// Below this much extracted text a page is a listing, not an article. @@ -29,6 +29,8 @@ impl App { match req.path.as_str() { "/" => self.home(), "/read" => self.read(req), + "/search" => self.search(req), + "/feed" => self.feed(req), "/go" => self.go(req), _ => Response::status(404, page::error("Not found", "No such page.", None)), } @@ -96,28 +98,39 @@ impl App { let page = match fetch::get(url, LIMIT) { Ok(p) => p, - Err(e) => { - return Response::status(502, page::error("Could not fetch", &e, Some(url))); - } + Err(e) => return Response::status(502, page::error("Could not fetch", &e, Some(url))), }; // A PDF or an image is not something to extract an article from. if !page.content_type.is_empty() && !page.content_type.contains("html") { + if feed::looks_like_feed(&page.content_type, &page.html) { + return Response::redirect(&format!("/feed?u={}", encode(&page.url))); + } return Response::redirect(&page.url); } + if feed::looks_like_feed(&page.content_type, &page.html) { + return Response::redirect(&format!("/feed?u={}", encode(&page.url))); + } let (doc, base) = follow_refreshes(page); - let article = extract::extract(&doc); - let body = render::body(&doc, &article, &base, &render::Options { - images: true, - link_prefix: Some("/read?u=".to_string()), - }); - - let html = self.article_page(&article, &base, &body); + let html = self.render_page(&doc, &base); self.cache.put(&key, &html); Response::html(html) } - fn article_page(&self, article: &extract::Article, base: &str, body: &str) -> String { + fn render_page(&self, doc: &Html, base: &str) -> String { + // A site we know how to read structurally beats generic extraction. + if let Some(view) = sites::adapt(doc, base) { + let content = format!("

{}

\n

{}

\n{}", + page::escape(&view.title), page::escape(&host_of(base)), view.body); + return page::shell(&view.title, "", &page::source_actions(base), &content); + } + + let article = extract::extract(doc); + let body = render::body(doc, &article, base, &render::Options { + images: true, + link_prefix: Some("/read?u=".to_string()), + }); + let mut meta = Vec::new(); if let Some(b) = &article.byline { meta.push(page::escape(b)); @@ -126,27 +139,141 @@ impl App { meta.push(page::escape(s)); } meta.push(page::escape(&host_of(base))); + if let Some(url) = sites::declared_feed(doc, base) { + meta.push(format!("feed", encode(&url))); + } let title = if article.title.is_empty() { host_of(base) } else { article.title.clone() }; - let mut content = format!( - "

{}

\n

{}

\n", - page::escape(&title), - meta.join(" · ") - ); + let mut content = + format!("

{}

\n

{}

\n", page::escape(&title), meta.join(" · ")); - if text_len(body) < THIN { - content.push_str(&page::notice( - "", - &format!( - "Not much of an article here. Try the original \ - or open it in the browser.", - page::escape(base), - encode(base) - ), + // Too little prose to be an article: show the page's structure instead. + if text_len(&body) < THIN { + let index = sites::index(doc, base); + if index.is_empty() { + content.push_str(&page::notice( + "", + &format!( + "Not much here. Try the original or \ + open it in the browser.", + page::escape(base), + encode(base) + ), + )); + content.push_str(&body); + } else { + content.push_str(&page::notice("", "No article on this page, so here are its links.")); + content.push_str(&index); + } + } else { + content.push_str(&body); + } + page::shell(&title, "", &page::source_actions(base), &content) + } + + fn search(&self, req: &Request) -> Response { + let Some(query) = req.param("q").filter(|q| !q.trim().is_empty()) else { + return Response::redirect("/"); + }; + let key = format!("search:{query}"); + if let Some(hit) = self.cache.get(&key) { + return Response::html(hit); + } + + let engine = Home::load().search.url; + let hits = match search::search(&engine, query) { + Ok(search::Outcome::Hits(h)) => h, + Ok(search::Outcome::Blocked) => { + let engine_host = host_of(&engine); + return Response::status(502, page::error( + "Search engine refused", + &format!( + "{engine_host} served a challenge page instead of results. \ + Point [search] url in the config at another HTML endpoint, \ + such as a SearXNG instance." + ), + Some(&engine.replace("{q}", &encode(query))), + )); + } + Err(e) => return Response::status(502, page::error("Search failed", &e, None)), + }; + + let mut body = format!("

{}

", page::escape(query)); + if hits.is_empty() { + body.push_str(&page::notice("", "No results.")); + } else { + body.push_str("
    "); + for h in &hits { + body.push_str(&format!( + "
  • {}{}{}
    • ", + page::read_link(&h.url), + page::escape(&h.title), + page::escape(&host_of(&h.url)), + if h.snippet.is_empty() { + String::new() + } else { + format!("{}", page::escape(&h.snippet)) + } + )); + } + body.push_str("
"); + } + + let html = page::shell(&format!("{query} — search"), query, "", &body); + if !hits.is_empty() { + self.cache.put(&key, &html); + } + Response::html(html) + } + + fn feed(&self, req: &Request) -> Response { + let Some(url) = req.param("u").filter(|u| !u.is_empty()) else { + return Response::redirect("/"); + }; + let key = format!("feed:{url}"); + if req.param("fresh").is_none() + && let Some(hit) = self.cache.get(&key) + { + return Response::html(hit); + } + + let page = match fetch::get(url, LIMIT) { + Ok(p) => p, + Err(e) => return Response::status(502, page::error("Could not fetch", &e, Some(url))), + }; + let Some(parsed) = feed::parse(&page.html) else { + return Response::status( + 502, + page::error("Not a feed", "No RSS items or Atom entries in that document.", Some(url)), + ); + }; + + let title = if parsed.title.is_empty() { host_of(url) } else { parsed.title.clone() }; + let mut body = format!( + "

{}

{} · {} entries

    ", + page::escape(&title), + page::escape(&host_of(url)), + parsed.entries.len() + ); + for e in &parsed.entries { + let label = if e.title.is_empty() { host_of(&e.link) } else { e.title.clone() }; + body.push_str(&format!( + "
  • {}{}{}
    • ", + page::read_link(&e.link), + page::escape(&label), + page::escape(&e.date), + if e.summary.is_empty() { + String::new() + } else { + format!("{}", page::escape(&truncate(&e.summary, 280))) + } )); } - content.push_str(body); - page::shell(&title, "", &page::source_actions(base), &content) + body.push_str("
"); + + let html = page::shell(&title, "", &page::source_actions(url), &body); + self.cache.put(&key, &html); + Response::html(html) } fn go(&self, req: &Request) -> Response { @@ -215,6 +342,15 @@ pub fn text_len(html: &str) -> usize { count } +fn truncate(s: &str, max: usize) -> String { + if s.chars().count() <= max { + return s.to_string(); + } + let cut: String = s.chars().take(max).collect(); + let at = cut.rfind(' ').unwrap_or(cut.len()); + format!("{}…", &cut[..at]) +} + pub fn host_of(url: &str) -> String { let rest = url.split_once("://").map(|(_, r)| r).unwrap_or(url); let authority = rest.split(['/', '?', '#']).next().unwrap_or(rest); diff --git a/furst-serve/src/main.rs b/furst-serve/src/main.rs index f0f5504..35a5a06 100644 --- a/furst-serve/src/main.rs +++ b/furst-serve/src/main.rs @@ -1,8 +1,11 @@ mod cache; mod config; +mod feed; mod handler; mod http; mod page; +mod search; +mod sites; use std::env; use std::io::Write; diff --git a/furst-serve/src/search.rs b/furst-serve/src/search.rs new file mode 100644 index 0000000..e636704 --- /dev/null +++ b/furst-serve/src/search.rs @@ -0,0 +1,238 @@ +//! Search, rendered as our own list. A results page costs a few kilobytes +//! instead of a megabyte. +//! +//! The engine is configurable because every free HTML endpoint eventually +//! rate-limits a repeat visitor, and being told so beats an empty page. + +use std::sync::LazyLock; + +use furst_read::urljoin::{decode, encode, join}; +use furst_read::{extract, fetch}; +use scraper::{ElementRef, Html, Selector}; + +const LIMIT: usize = 2 * 1024 * 1024; + +pub struct Hit { + pub title: String, + pub url: String, + pub snippet: String, +} + +pub enum Outcome { + Hits(Vec), + /// The engine served a challenge or an anomaly page instead of results. + Blocked, +} + +pub fn search(engine: &str, query: &str) -> Result { + let url = if engine.contains("{q}") { + engine.replace("{q}", &encode(query)) + } else { + format!("{engine}{}", encode(query)) + }; + let page = fetch::get(&url, LIMIT)?; + let doc = Html::parse_document(&page.html); + let hits = hits(&doc, &page.url); + + if hits.is_empty() && blocked(page.status, &page.html) { + return Ok(Outcome::Blocked); + } + Ok(Outcome::Hits(hits)) +} + +/// Engines answer a challenge with 2xx and a page that has no results on it, +/// so the body has to be inspected rather than the status alone. +fn blocked(status: u16, body: &str) -> bool { + if status != 200 { + return true; + } + let head = body[..body.len().min(20_000)].to_ascii_lowercase(); + [ + "anomaly", + "challenge", + "captcha", + "unusual traffic", + "are you a robot", + "verifying your browser", + "just a moment", + "enable javascript and cookies", + ] + .iter() + .any(|m| head.contains(m)) +} + +static DDG: LazyLock = LazyLock::new(|| Selector::parse("a.result__a").unwrap()); +static DDG_SNIP: LazyLock = LazyLock::new(|| Selector::parse(".result__snippet").unwrap()); +static SEARX: LazyLock = LazyLock::new(|| Selector::parse("article.result h3 a, .result h3 a").unwrap()); +static SEARX_SNIP: LazyLock = LazyLock::new(|| Selector::parse("p.content, .content").unwrap()); +static RESULT: LazyLock = LazyLock::new(|| Selector::parse("div.result, article.result, .result").unwrap()); +static ANY_LINK: LazyLock = LazyLock::new(|| Selector::parse("a[href]").unwrap()); +static HEADING_LINK: LazyLock = + LazyLock::new(|| Selector::parse("h1 a[href], h2 a[href], h3 a[href]").unwrap()); + +fn hits(doc: &Html, base: &str) -> Vec { + let structured = by_result_blocks(doc, base); + if !structured.is_empty() { + return structured; + } + // Unknown engine: reading the links off the page still works. + generic(doc, base) +} + +fn by_result_blocks(doc: &Html, base: &str) -> Vec { + let mut out = Vec::new(); + for result in doc.select(&RESULT) { + if result.value().attr("class").is_some_and(|c| c.contains("result--ad")) { + continue; + } + let Some(link) = result.select(&DDG).next().or_else(|| result.select(&SEARX).next()) else { + continue; + }; + let url = target(link.value().attr("href").unwrap_or(""), base); + if url.is_empty() { + continue; + } + let snippet = result + .select(&DDG_SNIP) + .next() + .or_else(|| result.select(&SEARX_SNIP).next()) + .map(text_of) + .unwrap_or_default(); + out.push(Hit { title: text_of(link), url, snippet }); + } + out +} + +fn generic(doc: &Html, base: &str) -> Vec { + // Result titles are headings on nearly every engine; footer chrome is not. + let headings = collect(doc.select(&HEADING_LINK), base); + if headings.len() >= 3 { + return headings; + } + let all = collect(doc.select(&ANY_LINK), base); + if all.len() > headings.len() { all } else { headings } +} + +fn collect<'a>(links: impl Iterator>, base: &str) -> Vec { + let engine_host = crate::handler::host_of(base); + let mut out: Vec = Vec::new(); + for link in links { + let title = text_of(link); + if title.chars().count() < 15 { + continue; + } + let url = target(link.value().attr("href").unwrap_or(""), base); + // Skip the engine's own navigation. + if url.is_empty() || crate::handler::host_of(&url) == engine_host { + continue; + } + if out.iter().any(|h| h.url == url) { + continue; + } + out.push(Hit { title, url, snippet: String::new() }); + if out.len() >= 40 { + break; + } + } + out +} + +/// Resolve a result href, unwrapping the redirect most engines wrap them in. +fn target(href: &str, base: &str) -> String { + let href = href.trim(); + if href.is_empty() || href.starts_with('#') { + return String::new(); + } + for key in ["uddg=", "url=", "u3=", "q="] { + if let Some(i) = href.find(key) { + let rest = &href[i + key.len()..]; + let end = rest.find('&').unwrap_or(rest.len()); + let decoded = decode(&rest[..end]); + if decoded.starts_with("http") { + return decoded; + } + } + } + let abs = join(base, href); + if abs.starts_with("http") { abs } else { String::new() } +} + +fn text_of(el: ElementRef) -> String { + extract::normalize(&extract::inner_text(el)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unwraps_the_redirect_wrapper() { + let b = "https://html.duckduckgo.com/html/"; + assert_eq!( + target("//duckduckgo.com/l/?uddg=https%3A%2F%2Fex.test%2Fa&rut=abc", b), + "https://ex.test/a" + ); + assert_eq!(target("https://ex.test/a", b), "https://ex.test/a"); + assert_eq!(target("#", b), ""); + } + + #[test] + fn reads_duckduckgo_results_and_skips_ads() { + let doc = Html::parse_document( + r#"
Ad
+
Title + A snippet
"#, + ); + let h = hits(&doc, "https://html.duckduckgo.com/html/"); + assert_eq!(h.len(), 1); + assert_eq!(h[0].title, "Title"); + assert_eq!(h[0].url, "https://ex.test/a"); + assert_eq!(h[0].snippet, "A snippet"); + } + + #[test] + fn reads_searxng_results() { + let doc = Html::parse_document( + r#""#, + ); + let h = hits(&doc, "https://searx.test/search"); + assert_eq!(h.len(), 1); + assert_eq!(h[0].url, "https://ex.test/b"); + assert_eq!(h[0].snippet, "Some content"); + } + + #[test] + fn generic_prefers_heading_links_over_footer_chrome() { + let doc = Html::parse_document( + r#"

First result title here

+

Second result title here

+

Third result title here

+ "#, + ); + let h = hits(&doc, "https://engine.test/search"); + assert_eq!(h.len(), 3); + assert!(h.iter().all(|x| !x.url.contains("engine-blog"))); + } + + #[test] + fn falls_back_to_links_for_an_unknown_engine() { + let doc = Html::parse_document( + r#"Prefs + About this engine + A result title long enough"#, + ); + let h = hits(&doc, "https://engine.test/search"); + assert_eq!(h.len(), 1, "engine's own pages and short links are skipped"); + assert_eq!(h[0].url, "https://ex.test/c"); + } + + #[test] + fn a_challenge_page_is_recognised() { + assert!(blocked(202, "")); + assert!(blocked(200, "anomaly detected")); + // What searx.be actually serves when it does not want to answer. + assert!(blocked(200, "Verifying your browser\u{2026}")); + assert!(!blocked(200, "normal results")); + } +} diff --git a/furst-serve/src/sites.rs b/furst-serve/src/sites.rs new file mode 100644 index 0000000..ff24d9c --- /dev/null +++ b/furst-serve/src/sites.rs @@ -0,0 +1,207 @@ +//! Views for pages that are not articles. +//! +//! Extraction looks for prose, so a front page or a comment thread correctly +//! yields almost nothing. These render the structure instead. + +use std::sync::LazyLock; + +use furst_read::urljoin::join; +use scraper::{ElementRef, Html, Selector}; + +use crate::page::{escape, read_link}; + +pub struct View { + pub title: String, + pub body: String, +} + +/// A site-specific view, when one applies. +pub fn adapt(doc: &Html, base: &str) -> Option { + if base.contains("news.ycombinator.com") { + return Some(hacker_news(doc, base)); + } + None +} + +// ---------------------------------------------------------------- Hacker News + +static HN_ROW: LazyLock = LazyLock::new(|| Selector::parse("tr.athing.submission").unwrap()); +static HN_TITLELINE: LazyLock = LazyLock::new(|| Selector::parse("span.titleline > a").unwrap()); +static HN_COMMENT: LazyLock = LazyLock::new(|| Selector::parse("tr.athing.comtr").unwrap()); +static HN_IND: LazyLock = LazyLock::new(|| Selector::parse("td.ind").unwrap()); +static HN_USER: LazyLock = LazyLock::new(|| Selector::parse("a.hnuser").unwrap()); +static HN_AGE: LazyLock = LazyLock::new(|| Selector::parse("span.age > a").unwrap()); +static HN_TEXT: LazyLock = LazyLock::new(|| Selector::parse(".commtext").unwrap()); +static HN_SUBTEXT: LazyLock = LazyLock::new(|| Selector::parse("td.subtext").unwrap()); + +fn hacker_news(doc: &Html, base: &str) -> View { + let stories = stories(doc, base); + let comments = comments(doc, base); + + let mut body = String::new(); + if !stories.is_empty() { + body.push_str(&stories); + } + if !comments.is_empty() { + if !stories.is_empty() { + body.push_str("

Comments

"); + } + body.push_str(&comments); + } + if body.is_empty() { + body.push_str("

Nothing to show on this page.

"); + } + View { title: "Hacker News".to_string(), body } +} + +fn stories(doc: &Html, base: &str) -> String { + let mut out = String::new(); + for row in doc.select(&HN_ROW) { + let Some(link) = row.select(&HN_TITLELINE).next() else { continue }; + let title = text_of(link); + let target = join(base, link.value().attr("href").unwrap_or("")); + + // The score and comment count live in the row after the title. + let sub = row + .next_siblings() + .filter_map(ElementRef::wrap) + .find_map(|s| s.select(&HN_SUBTEXT).next()); + let mut meta = Vec::new(); + let mut discussion = None; + if let Some(sub) = sub { + let score = sub + .select(&Selector::parse("span.score").unwrap()) + .next() + .map(text_of) + .unwrap_or_default(); + if !score.is_empty() { + meta.push(escape(&score)); + } + if let Some(user) = sub.select(&HN_USER).next() { + meta.push(escape(&text_of(user))); + } + // The discussion link is the last one, reading "N comments". + for a in sub.select(&Selector::parse("a").unwrap()) { + let t = text_of(a); + if t.contains("comment") || t == "discuss" { + discussion = Some((join(base, a.value().attr("href").unwrap_or("")), t)); + } + } + } + if let Some((_, label)) = &discussion { + meta.push(escape(label)); + } + + out.push_str(&format!( + "
  • {}{}{}
    • ", + read_link(&target), + escape(&title), + escape(&crate::handler::host_of(&target)), + if meta.is_empty() { String::new() } else { format!(" · {}", meta.join(" · ")) } + )); + if let Some((url, label)) = discussion { + out.push_str(&format!( + "
  • → {}
    • ", + read_link(&url), + escape(&label) + )); + } + } + if out.is_empty() { out } else { format!("
      {out}
    ") } +} + +fn comments(doc: &Html, base: &str) -> String { + let mut out = String::new(); + for row in doc.select(&HN_COMMENT) { + let depth: usize = row + .select(&HN_IND) + .next() + .and_then(|td| td.value().attr("indent")) + .and_then(|d| d.parse().ok()) + .unwrap_or(0); + let user = row.select(&HN_USER).next().map(text_of).unwrap_or_default(); + let age = row.select(&HN_AGE).next().map(text_of).unwrap_or_default(); + let Some(text) = row.select(&HN_TEXT).next() else { continue }; + + // Reuse the article renderer so links inside comments route back + // through the reader like every other link does. + let body = furst_read::render::body( + doc, + &furst_read::extract::Article { + title: String::new(), + byline: None, + site: None, + nodes: vec![text.id()], + }, + base, + &furst_read::render::Options { images: false, link_prefix: Some("/read?u=".to_string()) }, + ); + out.push_str(&format!( + "
    {} · {}
    {}
    ", + (depth.min(12) as f32) * 1.1, + escape(&user), + escape(&age), + body + )); + } + out +} + +// ------------------------------------------------------------- generic index + +static LINKS: LazyLock = LazyLock::new(|| Selector::parse("a[href]").unwrap()); + +/// The fallback for a page with no article: every link on it worth following. +pub fn index(doc: &Html, base: &str) -> String { + let mut seen: Vec = Vec::new(); + let mut rows = String::new(); + + for a in doc.select(&LINKS) { + let href = a.value().attr("href").unwrap_or("").trim(); + if href.is_empty() || href.starts_with('#') || href.to_ascii_lowercase().starts_with("javascript:") { + continue; + } + let text = text_of(a); + // Navigation is short; things worth reading are not. + if text.chars().count() < 18 { + continue; + } + let target = join(base, href); + if !target.starts_with("http") || target == base || seen.contains(&target) { + continue; + } + seen.push(target.clone()); + rows.push_str(&format!( + "
  • {}{}
    • ", + read_link(&target), + escape(&text), + escape(&crate::handler::host_of(&target)) + )); + if seen.len() >= 200 { + break; + } + } + + if rows.is_empty() { + String::new() + } else { + format!("

    {} links on this page

      {rows}
    ", seen.len()) + } +} + +/// A feed the page declares for itself, so the reader can offer it. +pub fn declared_feed(doc: &Html, base: &str) -> Option { + static ALT: LazyLock = + LazyLock::new(|| Selector::parse("link[rel~=\"alternate\"][href]").unwrap()); + for link in doc.select(&ALT) { + let kind = link.value().attr("type").unwrap_or(""); + if kind.contains("rss") || kind.contains("atom") { + return Some(join(base, link.value().attr("href").unwrap_or(""))); + } + } + None +} + +fn text_of(el: ElementRef) -> String { + furst_read::extract::normalize(&furst_read::extract::inner_text(el)) +}