diff --git a/furst-read/src/fetch.rs b/furst-read/src/fetch.rs index 40cd0b9..9bb3284 100644 --- a/furst-read/src/fetch.rs +++ b/furst-read/src/fetch.rs @@ -11,6 +11,7 @@ pub struct Page { pub url: String, pub html: String, pub content_type: String, + pub status: u16, } pub fn get(url: &str, limit: usize) -> Result { @@ -22,6 +23,7 @@ pub fn get(url: &str, limit: usize) -> Result { .into(); let mut resp = agent.get(url).call().map_err(|e| format!("{url}: {e}"))?; + let status = resp.status().as_u16(); let final_url = resp.get_uri().to_string(); let content_type = resp .headers() @@ -37,7 +39,7 @@ pub fn get(url: &str, limit: usize) -> Result { .read_to_vec() .map_err(|e| format!("{url}: {e}"))?; - Ok(Page { url: final_url, html: decode(&bytes, &content_type), content_type }) + Ok(Page { url: final_url, html: decode(&bytes, &content_type), content_type, status }) } /// Decode to text using, in order: the Content-Type charset, a `` diff --git a/furst-read/src/main.rs b/furst-read/src/main.rs index 54fc99d..f3daf02 100644 --- a/furst-read/src/main.rs +++ b/furst-read/src/main.rs @@ -78,7 +78,7 @@ fn run() -> Result<(), String> { let page = if stdin { let mut buf = Vec::new(); io::stdin().read_to_end(&mut buf).map_err(|e| e.to_string())?; - fetch::Page { url: url.clone(), html: fetch::decode(&buf, ""), content_type: String::new() } + fetch::Page { url: url.clone(), html: fetch::decode(&buf, ""), content_type: String::new(), status: 200 } } else { fetch::get(&url, LIMIT)? }; diff --git a/furst-serve/src/config.rs b/furst-serve/src/config.rs index 3479dc9..bbf5d03 100644 --- a/furst-serve/src/config.rs +++ b/furst-serve/src/config.rs @@ -12,6 +12,21 @@ pub struct Home { pub links: Vec, #[serde(default, rename = "feed")] pub feeds: Vec, + #[serde(default)] + pub search: Search, +} + +#[derive(Debug, Deserialize)] +pub struct Search { + /// Result-page URL with {q} where the encoded query goes. Any engine that + /// serves plain HTML works; the parser falls back to reading links. + pub url: String, +} + +impl Default for Search { + fn default() -> Self { + Search { url: "https://html.duckduckgo.com/html/?q={q}".to_string() } + } } #[derive(Debug, Deserialize)] @@ -63,4 +78,11 @@ url = "https://lwn.net/headlines/newrss" [[feed]] name = "Phoronix" url = "https://www.phoronix.com/rss.php" + +# Any engine that serves plain HTML. {q} is the encoded query. +# DuckDuckGo rate-limits repeat visitors; a SearXNG instance does not. +[search] +url = "https://html.duckduckgo.com/html/?q={q}" +# url = "https://searx.be/search?q={q}" +# url = "https://search.marginalia.nu/search?query={q}" "#; diff --git a/furst-serve/src/feed.rs b/furst-serve/src/feed.rs new file mode 100644 index 0000000..860d289 --- /dev/null +++ b/furst-serve/src/feed.rs @@ -0,0 +1,238 @@ +//! RSS and Atom, scanned rather than parsed. +//! +//! A feed only ever needs five fields per entry, and an HTML parser mangles +//! XML, so this walks the tags directly. No XML dependency, no surprises. + +pub struct Entry { + pub title: String, + pub link: String, + pub date: String, + pub summary: String, +} + +pub struct Feed { + pub title: String, + pub entries: Vec, +} + +pub fn looks_like_feed(content_type: &str, body: &str) -> bool { + let ct = content_type.to_ascii_lowercase(); + if ct.contains("rss") || ct.contains("atom") || ct.contains("xml") { + return true; + } + let head = &body[..body.len().min(1024)].to_ascii_lowercase(); + head.contains(" Option { + let mut entries = Vec::new(); + let mut cursor = 0; + let mut first_item = xml.len(); + + while entries.len() < 200 { + let item = ["item", "entry"] + .iter() + .filter_map(|name| block(xml, name, cursor)) + .min_by_key(|(start, _, _)| *start); + let Some((start, end, block)) = item else { break }; + first_item = first_item.min(start); + cursor = end; + + let title = tag_text(block, "title").unwrap_or_default(); + let link = entry_link(block).unwrap_or_default(); + if title.is_empty() && link.is_empty() { + continue; + } + let date = ["pubDate", "published", "updated", "dc:date"] + .iter() + .find_map(|n| tag_text(block, n)) + .unwrap_or_default(); + let summary = ["description", "summary", "content"] + .iter() + .find_map(|n| tag_text(block, n)) + .map(|s| strip_tags(&s)) + .unwrap_or_default(); + + entries.push(Entry { title, link, date, summary }); + } + + if entries.is_empty() { + return None; + } + // The channel title is whichever comes before the first entry. + let title = tag_text(&xml[..first_item], "title").unwrap_or_default(); + Some(Feed { title, entries }) +} + +/// Atom puts the URL in an attribute and may list several; RSS uses text. +fn entry_link(block: &str) -> Option<String> { + if let Some(text) = tag_text(block, "link").filter(|s| !s.is_empty()) { + return Some(text); + } + let mut best: Option<String> = None; + let mut cursor = 0; + while let Some(i) = block[cursor..].find("<link") { + let start = cursor + i; + let end = block[start..].find('>').map(|e| start + e)?; + let tag = &block[start..end]; + cursor = end; + let Some(href) = attr(tag, "href") else { continue }; + let rel = attr(tag, "rel").unwrap_or_default(); + if rel.is_empty() || rel == "alternate" { + return Some(href); + } + best.get_or_insert(href); + } + best +} + +/// Locate `<name ...> ... </name>`, returning its bounds and inner text. +fn block<'a>(xml: &'a str, name: &str, from: usize) -> Option<(usize, usize, &'a str)> { + let open = format!("<{name}"); + let close = format!("</{name}>"); + let mut cursor = from; + loop { + let start = cursor + xml[cursor..].find(&open)?; + // `<item` must not match `<items`. + let after = xml[start + open.len()..].chars().next(); + if after.is_some_and(|c| c.is_ascii_alphanumeric() || c == '-' || c == ':') { + cursor = start + open.len(); + continue; + } + let body_start = start + xml[start..].find('>')? + 1; + let end = body_start + xml[body_start..].find(&close)?; + return Some((start, end + close.len(), &xml[body_start..end])); + } +} + +fn tag_text(block: &str, name: &str) -> Option<String> { + let (_, _, inner) = self::block(block, name, 0)?; + Some(decode_entities(strip_cdata(inner).trim())) +} + +fn attr(tag: &str, name: &str) -> Option<String> { + let key = format!("{name}="); + let i = tag.find(&key)? + key.len(); + let rest = &tag[i..]; + let quote = rest.chars().next()?; + if quote == '"' || quote == '\'' { + let end = rest[1..].find(quote)? + 1; + Some(decode_entities(&rest[1..end])) + } else { + let end = rest.find([' ', '>', '/']).unwrap_or(rest.len()); + Some(decode_entities(&rest[..end])) + } +} + +fn strip_cdata(s: &str) -> &str { + s.trim().strip_prefix("<![CDATA[").and_then(|r| r.strip_suffix("]]>")).unwrap_or(s) +} + +pub fn strip_tags(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut in_tag = false; + for c in s.chars() { + match c { + '<' => in_tag = true, + '>' => in_tag = false, + _ if !in_tag => out.push(c), + _ => {} + } + } + decode_entities(&out).split_whitespace().collect::<Vec<_>>().join(" ") +} + +fn decode_entities(s: &str) -> String { + if !s.contains('&') { + return s.to_string(); + } + let mut out = String::with_capacity(s.len()); + let mut rest = s; + while let Some(i) = rest.find('&') { + out.push_str(&rest[..i]); + let tail = &rest[i..]; + let Some(semi) = tail[..tail.len().min(12)].find(';') else { + out.push('&'); + rest = &tail[1..]; + continue; + }; + let name = &tail[1..semi]; + let decoded = match name { + "amp" => Some('&'), + "lt" => Some('<'), + "gt" => Some('>'), + "quot" => Some('"'), + "apos" => Some('\''), + "nbsp" => Some(' '), + n if n.starts_with("#x") || n.starts_with("#X") => { + u32::from_str_radix(&n[2..], 16).ok().and_then(char::from_u32) + } + n if n.starts_with('#') => n[1..].parse().ok().and_then(char::from_u32), + _ => None, + }; + match decoded { + Some(c) => out.push(c), + None => out.push_str(&tail[..=semi]), + } + rest = &tail[semi + 1..]; + } + out.push_str(rest); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + const RSS: &str = r#"<?xml version="1.0"?><rss><channel> + <title>My Feedhttps://ex.test/ + First & besthttps://ex.test/1 + Mon, 01 Jan 2026 + Hello there

]]>
+ Secondhttps://ex.test/2 + "#; + + const ATOM: &str = r#"Atom Feed + + Entry one + + + 2026-01-01A summary"#; + + #[test] + fn parses_rss_with_cdata_and_entities() { + let f = parse(RSS).unwrap(); + assert_eq!(f.title, "My Feed"); + assert_eq!(f.entries.len(), 2); + assert_eq!(f.entries[0].title, "First & best"); + assert_eq!(f.entries[0].link, "https://ex.test/1"); + assert_eq!(f.entries[0].summary, "Hello there"); + assert_eq!(f.entries[1].title, "Second"); + } + + #[test] + fn atom_prefers_the_alternate_link_over_self() { + let f = parse(ATOM).unwrap(); + assert_eq!(f.title, "Atom Feed"); + assert_eq!(f.entries[0].link, "https://ex.test/one"); + assert_eq!(f.entries[0].summary, "A summary"); + } + + #[test] + fn detects_feeds_by_type_or_shape() { + assert!(looks_like_feed("application/rss+xml", "")); + assert!(looks_like_feed("", "")); + assert!(!looks_like_feed("text/html", "")); + } + + #[test] + fn a_document_with_no_items_is_not_a_feed() { + assert!(parse("

hi

").is_none()); + } + + #[test] + fn numeric_entities_decode() { + assert_eq!(decode_entities("a'b — c"), "a'b — c"); + assert_eq!(decode_entities("100% ¬anentity"), "100% ¬anentity"); + } +} diff --git a/furst-serve/src/handler.rs b/furst-serve/src/handler.rs index 01f67ca..79a5fb7 100644 --- a/furst-serve/src/handler.rs +++ b/furst-serve/src/handler.rs @@ -10,7 +10,7 @@ use scraper::Html; use crate::cache::Cache; use crate::config::Home; use crate::http::{Request, Response}; -use crate::page; +use crate::{feed, page, search, sites}; const LIMIT: usize = 8 * 1024 * 1024; /// Below this much extracted text a page is a listing, not an article. @@ -29,6 +29,8 @@ impl App { match req.path.as_str() { "/" => self.home(), "/read" => self.read(req), + "/search" => self.search(req), + "/feed" => self.feed(req), "/go" => self.go(req), _ => Response::status(404, page::error("Not found", "No such page.", None)), } @@ -96,28 +98,39 @@ impl App { let page = match fetch::get(url, LIMIT) { Ok(p) => p, - Err(e) => { - return Response::status(502, page::error("Could not fetch", &e, Some(url))); - } + Err(e) => return Response::status(502, page::error("Could not fetch", &e, Some(url))), }; // A PDF or an image is not something to extract an article from. if !page.content_type.is_empty() && !page.content_type.contains("html") { + if feed::looks_like_feed(&page.content_type, &page.html) { + return Response::redirect(&format!("/feed?u={}", encode(&page.url))); + } return Response::redirect(&page.url); } + if feed::looks_like_feed(&page.content_type, &page.html) { + return Response::redirect(&format!("/feed?u={}", encode(&page.url))); + } let (doc, base) = follow_refreshes(page); - let article = extract::extract(&doc); - let body = render::body(&doc, &article, &base, &render::Options { - images: true, - link_prefix: Some("/read?u=".to_string()), - }); - - let html = self.article_page(&article, &base, &body); + let html = self.render_page(&doc, &base); self.cache.put(&key, &html); Response::html(html) } - fn article_page(&self, article: &extract::Article, base: &str, body: &str) -> String { + fn render_page(&self, doc: &Html, base: &str) -> String { + // A site we know how to read structurally beats generic extraction. + if let Some(view) = sites::adapt(doc, base) { + let content = format!("

{}

\n

{}

\n{}", + page::escape(&view.title), page::escape(&host_of(base)), view.body); + return page::shell(&view.title, "", &page::source_actions(base), &content); + } + + let article = extract::extract(doc); + let body = render::body(doc, &article, base, &render::Options { + images: true, + link_prefix: Some("/read?u=".to_string()), + }); + let mut meta = Vec::new(); if let Some(b) = &article.byline { meta.push(page::escape(b)); @@ -126,27 +139,141 @@ impl App { meta.push(page::escape(s)); } meta.push(page::escape(&host_of(base))); + if let Some(url) = sites::declared_feed(doc, base) { + meta.push(format!("feed", encode(&url))); + } let title = if article.title.is_empty() { host_of(base) } else { article.title.clone() }; - let mut content = format!( - "

{}

\n

{}

\n", - page::escape(&title), - meta.join(" · ") - ); + let mut content = + format!("

{}

\n

{}

\n", page::escape(&title), meta.join(" · ")); - if text_len(body) < THIN { - content.push_str(&page::notice( - "", - &format!( - "Not much of an article here. Try the original \ - or open it in the browser.", - page::escape(base), - encode(base) - ), + // Too little prose to be an article: show the page's structure instead. + if text_len(&body) < THIN { + let index = sites::index(doc, base); + if index.is_empty() { + content.push_str(&page::notice( + "", + &format!( + "Not much here. Try the original or \ + open it in the browser.", + page::escape(base), + encode(base) + ), + )); + content.push_str(&body); + } else { + content.push_str(&page::notice("", "No article on this page, so here are its links.")); + content.push_str(&index); + } + } else { + content.push_str(&body); + } + page::shell(&title, "", &page::source_actions(base), &content) + } + + fn search(&self, req: &Request) -> Response { + let Some(query) = req.param("q").filter(|q| !q.trim().is_empty()) else { + return Response::redirect("/"); + }; + let key = format!("search:{query}"); + if let Some(hit) = self.cache.get(&key) { + return Response::html(hit); + } + + let engine = Home::load().search.url; + let hits = match search::search(&engine, query) { + Ok(search::Outcome::Hits(h)) => h, + Ok(search::Outcome::Blocked) => { + let engine_host = host_of(&engine); + return Response::status(502, page::error( + "Search engine refused", + &format!( + "{engine_host} served a challenge page instead of results. \ + Point [search] url in the config at another HTML endpoint, \ + such as a SearXNG instance." + ), + Some(&engine.replace("{q}", &encode(query))), + )); + } + Err(e) => return Response::status(502, page::error("Search failed", &e, None)), + }; + + let mut body = format!("

{}

", page::escape(query)); + if hits.is_empty() { + body.push_str(&page::notice("", "No results.")); + } else { + body.push_str("
    "); + for h in &hits { + body.push_str(&format!( + "
  • {}{}{}
  • ", + page::read_link(&h.url), + page::escape(&h.title), + page::escape(&host_of(&h.url)), + if h.snippet.is_empty() { + String::new() + } else { + format!("{}", page::escape(&h.snippet)) + } + )); + } + body.push_str("
"); + } + + let html = page::shell(&format!("{query} — search"), query, "", &body); + if !hits.is_empty() { + self.cache.put(&key, &html); + } + Response::html(html) + } + + fn feed(&self, req: &Request) -> Response { + let Some(url) = req.param("u").filter(|u| !u.is_empty()) else { + return Response::redirect("/"); + }; + let key = format!("feed:{url}"); + if req.param("fresh").is_none() + && let Some(hit) = self.cache.get(&key) + { + return Response::html(hit); + } + + let page = match fetch::get(url, LIMIT) { + Ok(p) => p, + Err(e) => return Response::status(502, page::error("Could not fetch", &e, Some(url))), + }; + let Some(parsed) = feed::parse(&page.html) else { + return Response::status( + 502, + page::error("Not a feed", "No RSS items or Atom entries in that document.", Some(url)), + ); + }; + + let title = if parsed.title.is_empty() { host_of(url) } else { parsed.title.clone() }; + let mut body = format!( + "

{}

{} · {} entries

    ", + page::escape(&title), + page::escape(&host_of(url)), + parsed.entries.len() + ); + for e in &parsed.entries { + let label = if e.title.is_empty() { host_of(&e.link) } else { e.title.clone() }; + body.push_str(&format!( + "
  • {}{}{}
  • ", + page::read_link(&e.link), + page::escape(&label), + page::escape(&e.date), + if e.summary.is_empty() { + String::new() + } else { + format!("{}", page::escape(&truncate(&e.summary, 280))) + } )); } - content.push_str(body); - page::shell(&title, "", &page::source_actions(base), &content) + body.push_str("
"); + + let html = page::shell(&title, "", &page::source_actions(url), &body); + self.cache.put(&key, &html); + Response::html(html) } fn go(&self, req: &Request) -> Response { @@ -215,6 +342,15 @@ pub fn text_len(html: &str) -> usize { count } +fn truncate(s: &str, max: usize) -> String { + if s.chars().count() <= max { + return s.to_string(); + } + let cut: String = s.chars().take(max).collect(); + let at = cut.rfind(' ').unwrap_or(cut.len()); + format!("{}…", &cut[..at]) +} + pub fn host_of(url: &str) -> String { let rest = url.split_once("://").map(|(_, r)| r).unwrap_or(url); let authority = rest.split(['/', '?', '#']).next().unwrap_or(rest); diff --git a/furst-serve/src/main.rs b/furst-serve/src/main.rs index f0f5504..35a5a06 100644 --- a/furst-serve/src/main.rs +++ b/furst-serve/src/main.rs @@ -1,8 +1,11 @@ mod cache; mod config; +mod feed; mod handler; mod http; mod page; +mod search; +mod sites; use std::env; use std::io::Write; diff --git a/furst-serve/src/search.rs b/furst-serve/src/search.rs new file mode 100644 index 0000000..e636704 --- /dev/null +++ b/furst-serve/src/search.rs @@ -0,0 +1,238 @@ +//! Search, rendered as our own list. A results page costs a few kilobytes +//! instead of a megabyte. +//! +//! The engine is configurable because every free HTML endpoint eventually +//! rate-limits a repeat visitor, and being told so beats an empty page. + +use std::sync::LazyLock; + +use furst_read::urljoin::{decode, encode, join}; +use furst_read::{extract, fetch}; +use scraper::{ElementRef, Html, Selector}; + +const LIMIT: usize = 2 * 1024 * 1024; + +pub struct Hit { + pub title: String, + pub url: String, + pub snippet: String, +} + +pub enum Outcome { + Hits(Vec), + /// The engine served a challenge or an anomaly page instead of results. + Blocked, +} + +pub fn search(engine: &str, query: &str) -> Result { + let url = if engine.contains("{q}") { + engine.replace("{q}", &encode(query)) + } else { + format!("{engine}{}", encode(query)) + }; + let page = fetch::get(&url, LIMIT)?; + let doc = Html::parse_document(&page.html); + let hits = hits(&doc, &page.url); + + if hits.is_empty() && blocked(page.status, &page.html) { + return Ok(Outcome::Blocked); + } + Ok(Outcome::Hits(hits)) +} + +/// Engines answer a challenge with 2xx and a page that has no results on it, +/// so the body has to be inspected rather than the status alone. +fn blocked(status: u16, body: &str) -> bool { + if status != 200 { + return true; + } + let head = body[..body.len().min(20_000)].to_ascii_lowercase(); + [ + "anomaly", + "challenge", + "captcha", + "unusual traffic", + "are you a robot", + "verifying your browser", + "just a moment", + "enable javascript and cookies", + ] + .iter() + .any(|m| head.contains(m)) +} + +static DDG: LazyLock = LazyLock::new(|| Selector::parse("a.result__a").unwrap()); +static DDG_SNIP: LazyLock = LazyLock::new(|| Selector::parse(".result__snippet").unwrap()); +static SEARX: LazyLock = LazyLock::new(|| Selector::parse("article.result h3 a, .result h3 a").unwrap()); +static SEARX_SNIP: LazyLock = LazyLock::new(|| Selector::parse("p.content, .content").unwrap()); +static RESULT: LazyLock = LazyLock::new(|| Selector::parse("div.result, article.result, .result").unwrap()); +static ANY_LINK: LazyLock = LazyLock::new(|| Selector::parse("a[href]").unwrap()); +static HEADING_LINK: LazyLock = + LazyLock::new(|| Selector::parse("h1 a[href], h2 a[href], h3 a[href]").unwrap()); + +fn hits(doc: &Html, base: &str) -> Vec { + let structured = by_result_blocks(doc, base); + if !structured.is_empty() { + return structured; + } + // Unknown engine: reading the links off the page still works. + generic(doc, base) +} + +fn by_result_blocks(doc: &Html, base: &str) -> Vec { + let mut out = Vec::new(); + for result in doc.select(&RESULT) { + if result.value().attr("class").is_some_and(|c| c.contains("result--ad")) { + continue; + } + let Some(link) = result.select(&DDG).next().or_else(|| result.select(&SEARX).next()) else { + continue; + }; + let url = target(link.value().attr("href").unwrap_or(""), base); + if url.is_empty() { + continue; + } + let snippet = result + .select(&DDG_SNIP) + .next() + .or_else(|| result.select(&SEARX_SNIP).next()) + .map(text_of) + .unwrap_or_default(); + out.push(Hit { title: text_of(link), url, snippet }); + } + out +} + +fn generic(doc: &Html, base: &str) -> Vec { + // Result titles are headings on nearly every engine; footer chrome is not. + let headings = collect(doc.select(&HEADING_LINK), base); + if headings.len() >= 3 { + return headings; + } + let all = collect(doc.select(&ANY_LINK), base); + if all.len() > headings.len() { all } else { headings } +} + +fn collect<'a>(links: impl Iterator>, base: &str) -> Vec { + let engine_host = crate::handler::host_of(base); + let mut out: Vec = Vec::new(); + for link in links { + let title = text_of(link); + if title.chars().count() < 15 { + continue; + } + let url = target(link.value().attr("href").unwrap_or(""), base); + // Skip the engine's own navigation. + if url.is_empty() || crate::handler::host_of(&url) == engine_host { + continue; + } + if out.iter().any(|h| h.url == url) { + continue; + } + out.push(Hit { title, url, snippet: String::new() }); + if out.len() >= 40 { + break; + } + } + out +} + +/// Resolve a result href, unwrapping the redirect most engines wrap them in. +fn target(href: &str, base: &str) -> String { + let href = href.trim(); + if href.is_empty() || href.starts_with('#') { + return String::new(); + } + for key in ["uddg=", "url=", "u3=", "q="] { + if let Some(i) = href.find(key) { + let rest = &href[i + key.len()..]; + let end = rest.find('&').unwrap_or(rest.len()); + let decoded = decode(&rest[..end]); + if decoded.starts_with("http") { + return decoded; + } + } + } + let abs = join(base, href); + if abs.starts_with("http") { abs } else { String::new() } +} + +fn text_of(el: ElementRef) -> String { + extract::normalize(&extract::inner_text(el)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unwraps_the_redirect_wrapper() { + let b = "https://html.duckduckgo.com/html/"; + assert_eq!( + target("//duckduckgo.com/l/?uddg=https%3A%2F%2Fex.test%2Fa&rut=abc", b), + "https://ex.test/a" + ); + assert_eq!(target("https://ex.test/a", b), "https://ex.test/a"); + assert_eq!(target("#", b), ""); + } + + #[test] + fn reads_duckduckgo_results_and_skips_ads() { + let doc = Html::parse_document( + r#" + "#, + ); + let h = hits(&doc, "https://html.duckduckgo.com/html/"); + assert_eq!(h.len(), 1); + assert_eq!(h[0].title, "Title"); + assert_eq!(h[0].url, "https://ex.test/a"); + assert_eq!(h[0].snippet, "A snippet"); + } + + #[test] + fn reads_searxng_results() { + let doc = Html::parse_document( + r#""#, + ); + let h = hits(&doc, "https://searx.test/search"); + assert_eq!(h.len(), 1); + assert_eq!(h[0].url, "https://ex.test/b"); + assert_eq!(h[0].snippet, "Some content"); + } + + #[test] + fn generic_prefers_heading_links_over_footer_chrome() { + let doc = Html::parse_document( + r#"

First result title here

+

Second result title here

+

Third result title here

+ "#, + ); + let h = hits(&doc, "https://engine.test/search"); + assert_eq!(h.len(), 3); + assert!(h.iter().all(|x| !x.url.contains("engine-blog"))); + } + + #[test] + fn falls_back_to_links_for_an_unknown_engine() { + let doc = Html::parse_document( + r#"Prefs + About this engine + A result title long enough"#, + ); + let h = hits(&doc, "https://engine.test/search"); + assert_eq!(h.len(), 1, "engine's own pages and short links are skipped"); + assert_eq!(h[0].url, "https://ex.test/c"); + } + + #[test] + fn a_challenge_page_is_recognised() { + assert!(blocked(202, "")); + assert!(blocked(200, "anomaly detected")); + // What searx.be actually serves when it does not want to answer. + assert!(blocked(200, "Verifying your browser\u{2026}")); + assert!(!blocked(200, "normal results")); + } +} diff --git a/furst-serve/src/sites.rs b/furst-serve/src/sites.rs new file mode 100644 index 0000000..ff24d9c --- /dev/null +++ b/furst-serve/src/sites.rs @@ -0,0 +1,207 @@ +//! Views for pages that are not articles. +//! +//! Extraction looks for prose, so a front page or a comment thread correctly +//! yields almost nothing. These render the structure instead. + +use std::sync::LazyLock; + +use furst_read::urljoin::join; +use scraper::{ElementRef, Html, Selector}; + +use crate::page::{escape, read_link}; + +pub struct View { + pub title: String, + pub body: String, +} + +/// A site-specific view, when one applies. +pub fn adapt(doc: &Html, base: &str) -> Option { + if base.contains("news.ycombinator.com") { + return Some(hacker_news(doc, base)); + } + None +} + +// ---------------------------------------------------------------- Hacker News + +static HN_ROW: LazyLock = LazyLock::new(|| Selector::parse("tr.athing.submission").unwrap()); +static HN_TITLELINE: LazyLock = LazyLock::new(|| Selector::parse("span.titleline > a").unwrap()); +static HN_COMMENT: LazyLock = LazyLock::new(|| Selector::parse("tr.athing.comtr").unwrap()); +static HN_IND: LazyLock = LazyLock::new(|| Selector::parse("td.ind").unwrap()); +static HN_USER: LazyLock = LazyLock::new(|| Selector::parse("a.hnuser").unwrap()); +static HN_AGE: LazyLock = LazyLock::new(|| Selector::parse("span.age > a").unwrap()); +static HN_TEXT: LazyLock = LazyLock::new(|| Selector::parse(".commtext").unwrap()); +static HN_SUBTEXT: LazyLock = LazyLock::new(|| Selector::parse("td.subtext").unwrap()); + +fn hacker_news(doc: &Html, base: &str) -> View { + let stories = stories(doc, base); + let comments = comments(doc, base); + + let mut body = String::new(); + if !stories.is_empty() { + body.push_str(&stories); + } + if !comments.is_empty() { + if !stories.is_empty() { + body.push_str("

Comments

"); + } + body.push_str(&comments); + } + if body.is_empty() { + body.push_str("

Nothing to show on this page.

"); + } + View { title: "Hacker News".to_string(), body } +} + +fn stories(doc: &Html, base: &str) -> String { + let mut out = String::new(); + for row in doc.select(&HN_ROW) { + let Some(link) = row.select(&HN_TITLELINE).next() else { continue }; + let title = text_of(link); + let target = join(base, link.value().attr("href").unwrap_or("")); + + // The score and comment count live in the row after the title. + let sub = row + .next_siblings() + .filter_map(ElementRef::wrap) + .find_map(|s| s.select(&HN_SUBTEXT).next()); + let mut meta = Vec::new(); + let mut discussion = None; + if let Some(sub) = sub { + let score = sub + .select(&Selector::parse("span.score").unwrap()) + .next() + .map(text_of) + .unwrap_or_default(); + if !score.is_empty() { + meta.push(escape(&score)); + } + if let Some(user) = sub.select(&HN_USER).next() { + meta.push(escape(&text_of(user))); + } + // The discussion link is the last one, reading "N comments". + for a in sub.select(&Selector::parse("a").unwrap()) { + let t = text_of(a); + if t.contains("comment") || t == "discuss" { + discussion = Some((join(base, a.value().attr("href").unwrap_or("")), t)); + } + } + } + if let Some((_, label)) = &discussion { + meta.push(escape(label)); + } + + out.push_str(&format!( + "
  • {}{}{}
  • ", + read_link(&target), + escape(&title), + escape(&crate::handler::host_of(&target)), + if meta.is_empty() { String::new() } else { format!(" · {}", meta.join(" · ")) } + )); + if let Some((url, label)) = discussion { + out.push_str(&format!( + "
  • → {}
  • ", + read_link(&url), + escape(&label) + )); + } + } + if out.is_empty() { out } else { format!("
      {out}
    ") } +} + +fn comments(doc: &Html, base: &str) -> String { + let mut out = String::new(); + for row in doc.select(&HN_COMMENT) { + let depth: usize = row + .select(&HN_IND) + .next() + .and_then(|td| td.value().attr("indent")) + .and_then(|d| d.parse().ok()) + .unwrap_or(0); + let user = row.select(&HN_USER).next().map(text_of).unwrap_or_default(); + let age = row.select(&HN_AGE).next().map(text_of).unwrap_or_default(); + let Some(text) = row.select(&HN_TEXT).next() else { continue }; + + // Reuse the article renderer so links inside comments route back + // through the reader like every other link does. + let body = furst_read::render::body( + doc, + &furst_read::extract::Article { + title: String::new(), + byline: None, + site: None, + nodes: vec![text.id()], + }, + base, + &furst_read::render::Options { images: false, link_prefix: Some("/read?u=".to_string()) }, + ); + out.push_str(&format!( + "
    {} · {}
    {}
    ", + (depth.min(12) as f32) * 1.1, + escape(&user), + escape(&age), + body + )); + } + out +} + +// ------------------------------------------------------------- generic index + +static LINKS: LazyLock = LazyLock::new(|| Selector::parse("a[href]").unwrap()); + +/// The fallback for a page with no article: every link on it worth following. +pub fn index(doc: &Html, base: &str) -> String { + let mut seen: Vec = Vec::new(); + let mut rows = String::new(); + + for a in doc.select(&LINKS) { + let href = a.value().attr("href").unwrap_or("").trim(); + if href.is_empty() || href.starts_with('#') || href.to_ascii_lowercase().starts_with("javascript:") { + continue; + } + let text = text_of(a); + // Navigation is short; things worth reading are not. + if text.chars().count() < 18 { + continue; + } + let target = join(base, href); + if !target.starts_with("http") || target == base || seen.contains(&target) { + continue; + } + seen.push(target.clone()); + rows.push_str(&format!( + "
  • {}{}
  • ", + read_link(&target), + escape(&text), + escape(&crate::handler::host_of(&target)) + )); + if seen.len() >= 200 { + break; + } + } + + if rows.is_empty() { + String::new() + } else { + format!("

    {} links on this page

      {rows}
    ", seen.len()) + } +} + +/// A feed the page declares for itself, so the reader can offer it. +pub fn declared_feed(doc: &Html, base: &str) -> Option { + static ALT: LazyLock = + LazyLock::new(|| Selector::parse("link[rel~=\"alternate\"][href]").unwrap()); + for link in doc.select(&ALT) { + let kind = link.value().attr("type").unwrap_or(""); + if kind.contains("rss") || kind.contains("atom") { + return Some(join(base, link.value().attr("href").unwrap_or(""))); + } + } + None +} + +fn text_of(el: ElementRef) -> String { + furst_read::extract::normalize(&furst_read::extract::inner_text(el)) +}