diff --git a/furst-read/src/fetch.rs b/furst-read/src/fetch.rs
index 40cd0b9..9bb3284 100644
--- a/furst-read/src/fetch.rs
+++ b/furst-read/src/fetch.rs
@@ -11,6 +11,7 @@ pub struct Page {
pub url: String,
pub html: String,
pub content_type: String,
+ pub status: u16,
}
pub fn get(url: &str, limit: usize) -> Result {
@@ -22,6 +23,7 @@ pub fn get(url: &str, limit: usize) -> Result {
.into();
let mut resp = agent.get(url).call().map_err(|e| format!("{url}: {e}"))?;
+ let status = resp.status().as_u16();
let final_url = resp.get_uri().to_string();
let content_type = resp
.headers()
@@ -37,7 +39,7 @@ pub fn get(url: &str, limit: usize) -> Result {
.read_to_vec()
.map_err(|e| format!("{url}: {e}"))?;
- Ok(Page { url: final_url, html: decode(&bytes, &content_type), content_type })
+ Ok(Page { url: final_url, html: decode(&bytes, &content_type), content_type, status })
}
/// Decode to text using, in order: the Content-Type charset, a ` `
diff --git a/furst-read/src/main.rs b/furst-read/src/main.rs
index 54fc99d..f3daf02 100644
--- a/furst-read/src/main.rs
+++ b/furst-read/src/main.rs
@@ -78,7 +78,7 @@ fn run() -> Result<(), String> {
let page = if stdin {
let mut buf = Vec::new();
io::stdin().read_to_end(&mut buf).map_err(|e| e.to_string())?;
- fetch::Page { url: url.clone(), html: fetch::decode(&buf, ""), content_type: String::new() }
+ fetch::Page { url: url.clone(), html: fetch::decode(&buf, ""), content_type: String::new(), status: 200 }
} else {
fetch::get(&url, LIMIT)?
};
diff --git a/furst-serve/src/config.rs b/furst-serve/src/config.rs
index 3479dc9..bbf5d03 100644
--- a/furst-serve/src/config.rs
+++ b/furst-serve/src/config.rs
@@ -12,6 +12,21 @@ pub struct Home {
pub links: Vec,
#[serde(default, rename = "feed")]
pub feeds: Vec,
+ #[serde(default)]
+ pub search: Search,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct Search {
+ /// Result-page URL with {q} where the encoded query goes. Any engine that
+ /// serves plain HTML works; the parser falls back to reading links.
+ pub url: String,
+}
+
+impl Default for Search {
+ fn default() -> Self {
+ Search { url: "https://html.duckduckgo.com/html/?q={q}".to_string() }
+ }
}
#[derive(Debug, Deserialize)]
@@ -63,4 +78,11 @@ url = "https://lwn.net/headlines/newrss"
[[feed]]
name = "Phoronix"
url = "https://www.phoronix.com/rss.php"
+
+# Any engine that serves plain HTML. {q} is the encoded query.
+# DuckDuckGo rate-limits repeat visitors; a SearXNG instance does not.
+[search]
+url = "https://html.duckduckgo.com/html/?q={q}"
+# url = "https://searx.be/search?q={q}"
+# url = "https://search.marginalia.nu/search?query={q}"
"#;
diff --git a/furst-serve/src/feed.rs b/furst-serve/src/feed.rs
new file mode 100644
index 0000000..860d289
--- /dev/null
+++ b/furst-serve/src/feed.rs
@@ -0,0 +1,238 @@
+//! RSS and Atom, scanned rather than parsed.
+//!
+//! A feed only ever needs five fields per entry, and an HTML parser mangles
+//! XML, so this walks the tags directly. No XML dependency, no surprises.
+
+pub struct Entry {
+ pub title: String,
+ pub link: String,
+ pub date: String,
+ pub summary: String,
+}
+
+pub struct Feed {
+ pub title: String,
+ pub entries: Vec,
+}
+
+pub fn looks_like_feed(content_type: &str, body: &str) -> bool {
+ let ct = content_type.to_ascii_lowercase();
+ if ct.contains("rss") || ct.contains("atom") || ct.contains("xml") {
+ return true;
+ }
+ let head = &body[..body.len().min(1024)].to_ascii_lowercase();
+ head.contains(" Option {
+ let mut entries = Vec::new();
+ let mut cursor = 0;
+ let mut first_item = xml.len();
+
+ while entries.len() < 200 {
+ let item = ["item", "entry"]
+ .iter()
+ .filter_map(|name| block(xml, name, cursor))
+ .min_by_key(|(start, _, _)| *start);
+ let Some((start, end, block)) = item else { break };
+ first_item = first_item.min(start);
+ cursor = end;
+
+ let title = tag_text(block, "title").unwrap_or_default();
+ let link = entry_link(block).unwrap_or_default();
+ if title.is_empty() && link.is_empty() {
+ continue;
+ }
+ let date = ["pubDate", "published", "updated", "dc:date"]
+ .iter()
+ .find_map(|n| tag_text(block, n))
+ .unwrap_or_default();
+ let summary = ["description", "summary", "content"]
+ .iter()
+ .find_map(|n| tag_text(block, n))
+ .map(|s| strip_tags(&s))
+ .unwrap_or_default();
+
+ entries.push(Entry { title, link, date, summary });
+ }
+
+ if entries.is_empty() {
+ return None;
+ }
+ // The channel title is whichever comes before the first entry.
+ let title = tag_text(&xml[..first_item], "title").unwrap_or_default();
+ Some(Feed { title, entries })
+}
+
+/// Atom puts the URL in an attribute and may list several; RSS uses text.
+fn entry_link(block: &str) -> Option {
+ if let Some(text) = tag_text(block, "link").filter(|s| !s.is_empty()) {
+ return Some(text);
+ }
+ let mut best: Option = None;
+ let mut cursor = 0;
+ while let Some(i) = block[cursor..].find(" ').map(|e| start + e)?;
+ let tag = &block[start..end];
+ cursor = end;
+ let Some(href) = attr(tag, "href") else { continue };
+ let rel = attr(tag, "rel").unwrap_or_default();
+ if rel.is_empty() || rel == "alternate" {
+ return Some(href);
+ }
+ best.get_or_insert(href);
+ }
+ best
+}
+
+/// Locate ` ... `, returning its bounds and inner text.
+fn block<'a>(xml: &'a str, name: &str, from: usize) -> Option<(usize, usize, &'a str)> {
+ let open = format!("<{name}");
+ let close = format!("{name}>");
+ let mut cursor = from;
+ loop {
+ let start = cursor + xml[cursor..].find(&open)?;
+ // `- ')? + 1;
+ let end = body_start + xml[body_start..].find(&close)?;
+ return Some((start, end + close.len(), &xml[body_start..end]));
+ }
+}
+
+fn tag_text(block: &str, name: &str) -> Option
{
+ let (_, _, inner) = self::block(block, name, 0)?;
+ Some(decode_entities(strip_cdata(inner).trim()))
+}
+
+fn attr(tag: &str, name: &str) -> Option {
+ let key = format!("{name}=");
+ let i = tag.find(&key)? + key.len();
+ let rest = &tag[i..];
+ let quote = rest.chars().next()?;
+ if quote == '"' || quote == '\'' {
+ let end = rest[1..].find(quote)? + 1;
+ Some(decode_entities(&rest[1..end]))
+ } else {
+ let end = rest.find([' ', '>', '/']).unwrap_or(rest.len());
+ Some(decode_entities(&rest[..end]))
+ }
+}
+
+fn strip_cdata(s: &str) -> &str {
+ s.trim().strip_prefix("")).unwrap_or(s)
+}
+
+pub fn strip_tags(s: &str) -> String {
+ let mut out = String::with_capacity(s.len());
+ let mut in_tag = false;
+ for c in s.chars() {
+ match c {
+ '<' => in_tag = true,
+ '>' => in_tag = false,
+ _ if !in_tag => out.push(c),
+ _ => {}
+ }
+ }
+ decode_entities(&out).split_whitespace().collect::>().join(" ")
+}
+
+fn decode_entities(s: &str) -> String {
+ if !s.contains('&') {
+ return s.to_string();
+ }
+ let mut out = String::with_capacity(s.len());
+ let mut rest = s;
+ while let Some(i) = rest.find('&') {
+ out.push_str(&rest[..i]);
+ let tail = &rest[i..];
+ let Some(semi) = tail[..tail.len().min(12)].find(';') else {
+ out.push('&');
+ rest = &tail[1..];
+ continue;
+ };
+ let name = &tail[1..semi];
+ let decoded = match name {
+ "amp" => Some('&'),
+ "lt" => Some('<'),
+ "gt" => Some('>'),
+ "quot" => Some('"'),
+ "apos" => Some('\''),
+ "nbsp" => Some(' '),
+ n if n.starts_with("#x") || n.starts_with("#X") => {
+ u32::from_str_radix(&n[2..], 16).ok().and_then(char::from_u32)
+ }
+ n if n.starts_with('#') => n[1..].parse().ok().and_then(char::from_u32),
+ _ => None,
+ };
+ match decoded {
+ Some(c) => out.push(c),
+ None => out.push_str(&tail[..=semi]),
+ }
+ rest = &tail[semi + 1..];
+ }
+ out.push_str(rest);
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const RSS: &str = r#"
+ My Feed https://ex.test/
+ First & best https://ex.test/1
+ Mon, 01 Jan 2026
+ Hello there
]]>
+ Second https://ex.test/2
+ "#;
+
+ const ATOM: &str = r#"Atom Feed
+
+ Entry one
+
+
+ 2026-01-01 A summary "#;
+
+ #[test]
+ fn parses_rss_with_cdata_and_entities() {
+ let f = parse(RSS).unwrap();
+ assert_eq!(f.title, "My Feed");
+ assert_eq!(f.entries.len(), 2);
+ assert_eq!(f.entries[0].title, "First & best");
+ assert_eq!(f.entries[0].link, "https://ex.test/1");
+ assert_eq!(f.entries[0].summary, "Hello there");
+ assert_eq!(f.entries[1].title, "Second");
+ }
+
+ #[test]
+ fn atom_prefers_the_alternate_link_over_self() {
+ let f = parse(ATOM).unwrap();
+ assert_eq!(f.title, "Atom Feed");
+ assert_eq!(f.entries[0].link, "https://ex.test/one");
+ assert_eq!(f.entries[0].summary, "A summary");
+ }
+
+ #[test]
+ fn detects_feeds_by_type_or_shape() {
+ assert!(looks_like_feed("application/rss+xml", ""));
+ assert!(looks_like_feed("", ""));
+ assert!(!looks_like_feed("text/html", ""));
+ }
+
+ #[test]
+ fn a_document_with_no_items_is_not_a_feed() {
+ assert!(parse("hi
").is_none());
+ }
+
+ #[test]
+ fn numeric_entities_decode() {
+ assert_eq!(decode_entities("a'b — c"), "a'b — c");
+ assert_eq!(decode_entities("100% ¬anentity"), "100% ¬anentity");
+ }
+}
diff --git a/furst-serve/src/handler.rs b/furst-serve/src/handler.rs
index 01f67ca..79a5fb7 100644
--- a/furst-serve/src/handler.rs
+++ b/furst-serve/src/handler.rs
@@ -10,7 +10,7 @@ use scraper::Html;
use crate::cache::Cache;
use crate::config::Home;
use crate::http::{Request, Response};
-use crate::page;
+use crate::{feed, page, search, sites};
const LIMIT: usize = 8 * 1024 * 1024;
/// Below this much extracted text a page is a listing, not an article.
@@ -29,6 +29,8 @@ impl App {
match req.path.as_str() {
"/" => self.home(),
"/read" => self.read(req),
+ "/search" => self.search(req),
+ "/feed" => self.feed(req),
"/go" => self.go(req),
_ => Response::status(404, page::error("Not found", "No such page.", None)),
}
@@ -96,28 +98,39 @@ impl App {
let page = match fetch::get(url, LIMIT) {
Ok(p) => p,
- Err(e) => {
- return Response::status(502, page::error("Could not fetch", &e, Some(url)));
- }
+ Err(e) => return Response::status(502, page::error("Could not fetch", &e, Some(url))),
};
// A PDF or an image is not something to extract an article from.
if !page.content_type.is_empty() && !page.content_type.contains("html") {
+ if feed::looks_like_feed(&page.content_type, &page.html) {
+ return Response::redirect(&format!("/feed?u={}", encode(&page.url)));
+ }
return Response::redirect(&page.url);
}
+ if feed::looks_like_feed(&page.content_type, &page.html) {
+ return Response::redirect(&format!("/feed?u={}", encode(&page.url)));
+ }
let (doc, base) = follow_refreshes(page);
- let article = extract::extract(&doc);
- let body = render::body(&doc, &article, &base, &render::Options {
- images: true,
- link_prefix: Some("/read?u=".to_string()),
- });
-
- let html = self.article_page(&article, &base, &body);
+ let html = self.render_page(&doc, &base);
self.cache.put(&key, &html);
Response::html(html)
}
- fn article_page(&self, article: &extract::Article, base: &str, body: &str) -> String {
+ fn render_page(&self, doc: &Html, base: &str) -> String {
+ // A site we know how to read structurally beats generic extraction.
+ if let Some(view) = sites::adapt(doc, base) {
+ let content = format!("{} \n{}
\n{}",
+ page::escape(&view.title), page::escape(&host_of(base)), view.body);
+ return page::shell(&view.title, "", &page::source_actions(base), &content);
+ }
+
+ let article = extract::extract(doc);
+ let body = render::body(doc, &article, base, &render::Options {
+ images: true,
+ link_prefix: Some("/read?u=".to_string()),
+ });
+
let mut meta = Vec::new();
if let Some(b) = &article.byline {
meta.push(page::escape(b));
@@ -126,27 +139,141 @@ impl App {
meta.push(page::escape(s));
}
meta.push(page::escape(&host_of(base)));
+ if let Some(url) = sites::declared_feed(doc, base) {
+ meta.push(format!("feed ", encode(&url)));
+ }
let title = if article.title.is_empty() { host_of(base) } else { article.title.clone() };
- let mut content = format!(
- "{} \n{}
\n",
- page::escape(&title),
- meta.join(" · ")
- );
+ let mut content =
+ format!("{} \n{}
\n", page::escape(&title), meta.join(" · "));
- if text_len(body) < THIN {
- content.push_str(&page::notice(
- "",
- &format!(
- "Not much of an article here. Try the original \
- or open it in the browser .",
- page::escape(base),
- encode(base)
- ),
+ // Too little prose to be an article: show the page's structure instead.
+ if text_len(&body) < THIN {
+ let index = sites::index(doc, base);
+ if index.is_empty() {
+ content.push_str(&page::notice(
+ "",
+ &format!(
+ "Not much here. Try the original or \
+ open it in the browser .",
+ page::escape(base),
+ encode(base)
+ ),
+ ));
+ content.push_str(&body);
+ } else {
+ content.push_str(&page::notice("", "No article on this page, so here are its links."));
+ content.push_str(&index);
+ }
+ } else {
+ content.push_str(&body);
+ }
+ page::shell(&title, "", &page::source_actions(base), &content)
+ }
+
+ fn search(&self, req: &Request) -> Response {
+ let Some(query) = req.param("q").filter(|q| !q.trim().is_empty()) else {
+ return Response::redirect("/");
+ };
+ let key = format!("search:{query}");
+ if let Some(hit) = self.cache.get(&key) {
+ return Response::html(hit);
+ }
+
+ let engine = Home::load().search.url;
+ let hits = match search::search(&engine, query) {
+ Ok(search::Outcome::Hits(h)) => h,
+ Ok(search::Outcome::Blocked) => {
+ let engine_host = host_of(&engine);
+ return Response::status(502, page::error(
+ "Search engine refused",
+ &format!(
+ "{engine_host} served a challenge page instead of results. \
+ Point [search] url in the config at another HTML endpoint, \
+ such as a SearXNG instance."
+ ),
+ Some(&engine.replace("{q}", &encode(query))),
+ ));
+ }
+ Err(e) => return Response::status(502, page::error("Search failed", &e, None)),
+ };
+
+ let mut body = format!("{} ", page::escape(query));
+ if hits.is_empty() {
+ body.push_str(&page::notice("", "No results."));
+ } else {
+ body.push_str("");
+ for h in &hits {
+ body.push_str(&format!(
+ "{} {} {} ",
+ page::read_link(&h.url),
+ page::escape(&h.title),
+ page::escape(&host_of(&h.url)),
+ if h.snippet.is_empty() {
+ String::new()
+ } else {
+ format!("{} ", page::escape(&h.snippet))
+ }
+ ));
+ }
+ body.push_str(" ");
+ }
+
+ let html = page::shell(&format!("{query} — search"), query, "", &body);
+ if !hits.is_empty() {
+ self.cache.put(&key, &html);
+ }
+ Response::html(html)
+ }
+
+ fn feed(&self, req: &Request) -> Response {
+ let Some(url) = req.param("u").filter(|u| !u.is_empty()) else {
+ return Response::redirect("/");
+ };
+ let key = format!("feed:{url}");
+ if req.param("fresh").is_none()
+ && let Some(hit) = self.cache.get(&key)
+ {
+ return Response::html(hit);
+ }
+
+ let page = match fetch::get(url, LIMIT) {
+ Ok(p) => p,
+ Err(e) => return Response::status(502, page::error("Could not fetch", &e, Some(url))),
+ };
+ let Some(parsed) = feed::parse(&page.html) else {
+ return Response::status(
+ 502,
+ page::error("Not a feed", "No RSS items or Atom entries in that document.", Some(url)),
+ );
+ };
+
+ let title = if parsed.title.is_empty() { host_of(url) } else { parsed.title.clone() };
+ let mut body = format!(
+ "{} {} · {} entries
",
+ page::escape(&title),
+ page::escape(&host_of(url)),
+ parsed.entries.len()
+ );
+ for e in &parsed.entries {
+ let label = if e.title.is_empty() { host_of(&e.link) } else { e.title.clone() };
+ body.push_str(&format!(
+ "{} {} {} ",
+ page::read_link(&e.link),
+ page::escape(&label),
+ page::escape(&e.date),
+ if e.summary.is_empty() {
+ String::new()
+ } else {
+ format!("{} ", page::escape(&truncate(&e.summary, 280)))
+ }
));
}
- content.push_str(body);
- page::shell(&title, "", &page::source_actions(base), &content)
+ body.push_str(" ");
+
+ let html = page::shell(&title, "", &page::source_actions(url), &body);
+ self.cache.put(&key, &html);
+ Response::html(html)
}
fn go(&self, req: &Request) -> Response {
@@ -215,6 +342,15 @@ pub fn text_len(html: &str) -> usize {
count
}
+fn truncate(s: &str, max: usize) -> String {
+ if s.chars().count() <= max {
+ return s.to_string();
+ }
+ let cut: String = s.chars().take(max).collect();
+ let at = cut.rfind(' ').unwrap_or(cut.len());
+ format!("{}…", &cut[..at])
+}
+
pub fn host_of(url: &str) -> String {
let rest = url.split_once("://").map(|(_, r)| r).unwrap_or(url);
let authority = rest.split(['/', '?', '#']).next().unwrap_or(rest);
diff --git a/furst-serve/src/main.rs b/furst-serve/src/main.rs
index f0f5504..35a5a06 100644
--- a/furst-serve/src/main.rs
+++ b/furst-serve/src/main.rs
@@ -1,8 +1,11 @@
mod cache;
mod config;
+mod feed;
mod handler;
mod http;
mod page;
+mod search;
+mod sites;
use std::env;
use std::io::Write;
diff --git a/furst-serve/src/search.rs b/furst-serve/src/search.rs
new file mode 100644
index 0000000..e636704
--- /dev/null
+++ b/furst-serve/src/search.rs
@@ -0,0 +1,238 @@
+//! Search, rendered as our own list. A results page costs a few kilobytes
+//! instead of a megabyte.
+//!
+//! The engine is configurable because every free HTML endpoint eventually
+//! rate-limits a repeat visitor, and being told so beats an empty page.
+
+use std::sync::LazyLock;
+
+use furst_read::urljoin::{decode, encode, join};
+use furst_read::{extract, fetch};
+use scraper::{ElementRef, Html, Selector};
+
+const LIMIT: usize = 2 * 1024 * 1024;
+
+pub struct Hit {
+ pub title: String,
+ pub url: String,
+ pub snippet: String,
+}
+
+pub enum Outcome {
+ Hits(Vec),
+ /// The engine served a challenge or an anomaly page instead of results.
+ Blocked,
+}
+
+pub fn search(engine: &str, query: &str) -> Result {
+ let url = if engine.contains("{q}") {
+ engine.replace("{q}", &encode(query))
+ } else {
+ format!("{engine}{}", encode(query))
+ };
+ let page = fetch::get(&url, LIMIT)?;
+ let doc = Html::parse_document(&page.html);
+ let hits = hits(&doc, &page.url);
+
+ if hits.is_empty() && blocked(page.status, &page.html) {
+ return Ok(Outcome::Blocked);
+ }
+ Ok(Outcome::Hits(hits))
+}
+
+/// Engines answer a challenge with 2xx and a page that has no results on it,
+/// so the body has to be inspected rather than the status alone.
+fn blocked(status: u16, body: &str) -> bool {
+ if status != 200 {
+ return true;
+ }
+ let head = body[..body.len().min(20_000)].to_ascii_lowercase();
+ [
+ "anomaly",
+ "challenge",
+ "captcha",
+ "unusual traffic",
+ "are you a robot",
+ "verifying your browser",
+ "just a moment",
+ "enable javascript and cookies",
+ ]
+ .iter()
+ .any(|m| head.contains(m))
+}
+
+static DDG: LazyLock = LazyLock::new(|| Selector::parse("a.result__a").unwrap());
+static DDG_SNIP: LazyLock = LazyLock::new(|| Selector::parse(".result__snippet").unwrap());
+static SEARX: LazyLock = LazyLock::new(|| Selector::parse("article.result h3 a, .result h3 a").unwrap());
+static SEARX_SNIP: LazyLock = LazyLock::new(|| Selector::parse("p.content, .content").unwrap());
+static RESULT: LazyLock = LazyLock::new(|| Selector::parse("div.result, article.result, .result").unwrap());
+static ANY_LINK: LazyLock = LazyLock::new(|| Selector::parse("a[href]").unwrap());
+static HEADING_LINK: LazyLock =
+ LazyLock::new(|| Selector::parse("h1 a[href], h2 a[href], h3 a[href]").unwrap());
+
+fn hits(doc: &Html, base: &str) -> Vec {
+ let structured = by_result_blocks(doc, base);
+ if !structured.is_empty() {
+ return structured;
+ }
+ // Unknown engine: reading the links off the page still works.
+ generic(doc, base)
+}
+
+fn by_result_blocks(doc: &Html, base: &str) -> Vec {
+ let mut out = Vec::new();
+ for result in doc.select(&RESULT) {
+ if result.value().attr("class").is_some_and(|c| c.contains("result--ad")) {
+ continue;
+ }
+ let Some(link) = result.select(&DDG).next().or_else(|| result.select(&SEARX).next()) else {
+ continue;
+ };
+ let url = target(link.value().attr("href").unwrap_or(""), base);
+ if url.is_empty() {
+ continue;
+ }
+ let snippet = result
+ .select(&DDG_SNIP)
+ .next()
+ .or_else(|| result.select(&SEARX_SNIP).next())
+ .map(text_of)
+ .unwrap_or_default();
+ out.push(Hit { title: text_of(link), url, snippet });
+ }
+ out
+}
+
+fn generic(doc: &Html, base: &str) -> Vec {
+ // Result titles are headings on nearly every engine; footer chrome is not.
+ let headings = collect(doc.select(&HEADING_LINK), base);
+ if headings.len() >= 3 {
+ return headings;
+ }
+ let all = collect(doc.select(&ANY_LINK), base);
+ if all.len() > headings.len() { all } else { headings }
+}
+
+fn collect<'a>(links: impl Iterator- >, base: &str) -> Vec
{
+ let engine_host = crate::handler::host_of(base);
+ let mut out: Vec = Vec::new();
+ for link in links {
+ let title = text_of(link);
+ if title.chars().count() < 15 {
+ continue;
+ }
+ let url = target(link.value().attr("href").unwrap_or(""), base);
+ // Skip the engine's own navigation.
+ if url.is_empty() || crate::handler::host_of(&url) == engine_host {
+ continue;
+ }
+ if out.iter().any(|h| h.url == url) {
+ continue;
+ }
+ out.push(Hit { title, url, snippet: String::new() });
+ if out.len() >= 40 {
+ break;
+ }
+ }
+ out
+}
+
+/// Resolve a result href, unwrapping the redirect most engines wrap them in.
+fn target(href: &str, base: &str) -> String {
+ let href = href.trim();
+ if href.is_empty() || href.starts_with('#') {
+ return String::new();
+ }
+ for key in ["uddg=", "url=", "u3=", "q="] {
+ if let Some(i) = href.find(key) {
+ let rest = &href[i + key.len()..];
+ let end = rest.find('&').unwrap_or(rest.len());
+ let decoded = decode(&rest[..end]);
+ if decoded.starts_with("http") {
+ return decoded;
+ }
+ }
+ }
+ let abs = join(base, href);
+ if abs.starts_with("http") { abs } else { String::new() }
+}
+
+fn text_of(el: ElementRef) -> String {
+ extract::normalize(&extract::inner_text(el))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn unwraps_the_redirect_wrapper() {
+ let b = "https://html.duckduckgo.com/html/";
+ assert_eq!(
+ target("//duckduckgo.com/l/?uddg=https%3A%2F%2Fex.test%2Fa&rut=abc", b),
+ "https://ex.test/a"
+ );
+ assert_eq!(target("https://ex.test/a", b), "https://ex.test/a");
+ assert_eq!(target("#", b), "");
+ }
+
+ #[test]
+ fn reads_duckduckgo_results_and_skips_ads() {
+ let doc = Html::parse_document(
+ r#"
+ "#,
+ );
+ let h = hits(&doc, "https://html.duckduckgo.com/html/");
+ assert_eq!(h.len(), 1);
+ assert_eq!(h[0].title, "Title");
+ assert_eq!(h[0].url, "https://ex.test/a");
+ assert_eq!(h[0].snippet, "A snippet");
+ }
+
+ #[test]
+ fn reads_searxng_results() {
+ let doc = Html::parse_document(
+ r#"
+ Some content
"#,
+ );
+ let h = hits(&doc, "https://searx.test/search");
+ assert_eq!(h.len(), 1);
+ assert_eq!(h[0].url, "https://ex.test/b");
+ assert_eq!(h[0].snippet, "Some content");
+ }
+
+ #[test]
+ fn generic_prefers_heading_links_over_footer_chrome() {
+ let doc = Html::parse_document(
+ r#"
+
+
+ "#,
+ );
+ let h = hits(&doc, "https://engine.test/search");
+ assert_eq!(h.len(), 3);
+ assert!(h.iter().all(|x| !x.url.contains("engine-blog")));
+ }
+
+ #[test]
+ fn falls_back_to_links_for_an_unknown_engine() {
+ let doc = Html::parse_document(
+ r#"Prefs
+ About this engine
+ A result title long enough "#,
+ );
+ let h = hits(&doc, "https://engine.test/search");
+ assert_eq!(h.len(), 1, "engine's own pages and short links are skipped");
+ assert_eq!(h[0].url, "https://ex.test/c");
+ }
+
+ #[test]
+ fn a_challenge_page_is_recognised() {
+ assert!(blocked(202, ""));
+ assert!(blocked(200, "anomaly detected"));
+ // What searx.be actually serves when it does not want to answer.
+ assert!(blocked(200, "Verifying your browser\u{2026} "));
+ assert!(!blocked(200, "normal results"));
+ }
+}
diff --git a/furst-serve/src/sites.rs b/furst-serve/src/sites.rs
new file mode 100644
index 0000000..ff24d9c
--- /dev/null
+++ b/furst-serve/src/sites.rs
@@ -0,0 +1,207 @@
+//! Views for pages that are not articles.
+//!
+//! Extraction looks for prose, so a front page or a comment thread correctly
+//! yields almost nothing. These render the structure instead.
+
+use std::sync::LazyLock;
+
+use furst_read::urljoin::join;
+use scraper::{ElementRef, Html, Selector};
+
+use crate::page::{escape, read_link};
+
+pub struct View {
+ pub title: String,
+ pub body: String,
+}
+
+/// A site-specific view, when one applies.
+pub fn adapt(doc: &Html, base: &str) -> Option {
+ if base.contains("news.ycombinator.com") {
+ return Some(hacker_news(doc, base));
+ }
+ None
+}
+
+// ---------------------------------------------------------------- Hacker News
+
+static HN_ROW: LazyLock = LazyLock::new(|| Selector::parse("tr.athing.submission").unwrap());
+static HN_TITLELINE: LazyLock = LazyLock::new(|| Selector::parse("span.titleline > a").unwrap());
+static HN_COMMENT: LazyLock = LazyLock::new(|| Selector::parse("tr.athing.comtr").unwrap());
+static HN_IND: LazyLock = LazyLock::new(|| Selector::parse("td.ind").unwrap());
+static HN_USER: LazyLock = LazyLock::new(|| Selector::parse("a.hnuser").unwrap());
+static HN_AGE: LazyLock = LazyLock::new(|| Selector::parse("span.age > a").unwrap());
+static HN_TEXT: LazyLock = LazyLock::new(|| Selector::parse(".commtext").unwrap());
+static HN_SUBTEXT: LazyLock = LazyLock::new(|| Selector::parse("td.subtext").unwrap());
+
+fn hacker_news(doc: &Html, base: &str) -> View {
+ let stories = stories(doc, base);
+ let comments = comments(doc, base);
+
+ let mut body = String::new();
+ if !stories.is_empty() {
+ body.push_str(&stories);
+ }
+ if !comments.is_empty() {
+ if !stories.is_empty() {
+ body.push_str("Comments
");
+ }
+ body.push_str(&comments);
+ }
+ if body.is_empty() {
+ body.push_str("Nothing to show on this page.
");
+ }
+ View { title: "Hacker News".to_string(), body }
+}
+
+fn stories(doc: &Html, base: &str) -> String {
+ let mut out = String::new();
+ for row in doc.select(&HN_ROW) {
+ let Some(link) = row.select(&HN_TITLELINE).next() else { continue };
+ let title = text_of(link);
+ let target = join(base, link.value().attr("href").unwrap_or(""));
+
+ // The score and comment count live in the row after the title.
+ let sub = row
+ .next_siblings()
+ .filter_map(ElementRef::wrap)
+ .find_map(|s| s.select(&HN_SUBTEXT).next());
+ let mut meta = Vec::new();
+ let mut discussion = None;
+ if let Some(sub) = sub {
+ let score = sub
+ .select(&Selector::parse("span.score").unwrap())
+ .next()
+ .map(text_of)
+ .unwrap_or_default();
+ if !score.is_empty() {
+ meta.push(escape(&score));
+ }
+ if let Some(user) = sub.select(&HN_USER).next() {
+ meta.push(escape(&text_of(user)));
+ }
+ // The discussion link is the last one, reading "N comments".
+ for a in sub.select(&Selector::parse("a").unwrap()) {
+ let t = text_of(a);
+ if t.contains("comment") || t == "discuss" {
+ discussion = Some((join(base, a.value().attr("href").unwrap_or("")), t));
+ }
+ }
+ }
+ if let Some((_, label)) = &discussion {
+ meta.push(escape(label));
+ }
+
+ out.push_str(&format!(
+ "{} {}{} ",
+ read_link(&target),
+ escape(&title),
+ escape(&crate::handler::host_of(&target)),
+ if meta.is_empty() { String::new() } else { format!(" · {}", meta.join(" · ")) }
+ ));
+ if let Some((url, label)) = discussion {
+ out.push_str(&format!(
+ "→ {} ",
+ read_link(&url),
+ escape(&label)
+ ));
+ }
+ }
+ if out.is_empty() { out } else { format!("") }
+}
+
+fn comments(doc: &Html, base: &str) -> String {
+ let mut out = String::new();
+ for row in doc.select(&HN_COMMENT) {
+ let depth: usize = row
+ .select(&HN_IND)
+ .next()
+ .and_then(|td| td.value().attr("indent"))
+ .and_then(|d| d.parse().ok())
+ .unwrap_or(0);
+ let user = row.select(&HN_USER).next().map(text_of).unwrap_or_default();
+ let age = row.select(&HN_AGE).next().map(text_of).unwrap_or_default();
+ let Some(text) = row.select(&HN_TEXT).next() else { continue };
+
+ // Reuse the article renderer so links inside comments route back
+ // through the reader like every other link does.
+ let body = furst_read::render::body(
+ doc,
+ &furst_read::extract::Article {
+ title: String::new(),
+ byline: None,
+ site: None,
+ nodes: vec![text.id()],
+ },
+ base,
+ &furst_read::render::Options { images: false, link_prefix: Some("/read?u=".to_string()) },
+ );
+ out.push_str(&format!(
+ "",
+ (depth.min(12) as f32) * 1.1,
+ escape(&user),
+ escape(&age),
+ body
+ ));
+ }
+ out
+}
+
+// ------------------------------------------------------------- generic index
+
+static LINKS: LazyLock = LazyLock::new(|| Selector::parse("a[href]").unwrap());
+
+/// The fallback for a page with no article: every link on it worth following.
+pub fn index(doc: &Html, base: &str) -> String {
+ let mut seen: Vec = Vec::new();
+ let mut rows = String::new();
+
+ for a in doc.select(&LINKS) {
+ let href = a.value().attr("href").unwrap_or("").trim();
+ if href.is_empty() || href.starts_with('#') || href.to_ascii_lowercase().starts_with("javascript:") {
+ continue;
+ }
+ let text = text_of(a);
+ // Navigation is short; things worth reading are not.
+ if text.chars().count() < 18 {
+ continue;
+ }
+ let target = join(base, href);
+ if !target.starts_with("http") || target == base || seen.contains(&target) {
+ continue;
+ }
+ seen.push(target.clone());
+ rows.push_str(&format!(
+ "{} {} ",
+ read_link(&target),
+ escape(&text),
+ escape(&crate::handler::host_of(&target))
+ ));
+ if seen.len() >= 200 {
+ break;
+ }
+ }
+
+ if rows.is_empty() {
+ String::new()
+ } else {
+ format!("{} links on this page
", seen.len())
+ }
+}
+
+/// A feed the page declares for itself, so the reader can offer it.
+pub fn declared_feed(doc: &Html, base: &str) -> Option {
+ static ALT: LazyLock =
+ LazyLock::new(|| Selector::parse("link[rel~=\"alternate\"][href]").unwrap());
+ for link in doc.select(&ALT) {
+ let kind = link.value().attr("type").unwrap_or("");
+ if kind.contains("rss") || kind.contains("atom") {
+ return Some(join(base, link.value().attr("href").unwrap_or("")));
+ }
+ }
+ None
+}
+
+fn text_of(el: ElementRef) -> String {
+ furst_read::extract::normalize(&furst_read::extract::inner_text(el))
+}