Add search, feeds and listing views to the reader server
Extraction looks for prose, so a front page, a comment thread or a search
result page correctly yields almost nothing. These render the structure
instead, and every link they emit routes back through /read.
/search?q= results from a configurable HTML endpoint
/feed?u= RSS and Atom
/read?u= now picks a site view, falling back to a link index when a
page has too little text to be an article
Feeds are scanned rather than parsed. A feed needs five fields per entry
and an HTML parser mangles XML, so this walks the tags directly: CDATA,
named and numeric entities, and Atom's preference for rel=alternate over
rel=self. No XML dependency.
Hacker News gets a real adapter: stories with score, author and a link
into the discussion, and comment threads rendered with their indent
preserved. Comment bodies go through the article renderer so links inside
them behave like every other link.
Search is deliberately engine-agnostic. Known result shapes are tried
first, then heading links, then any link, because every free HTML endpoint
eventually rate-limits a repeat visitor. When one answers with a challenge
page rather than results the reader says so and points at the config,
instead of showing an empty page; detection reads the body, since these
arrive as 200 or 202 rather than an error status. A blocked search is
never cached.
Pages that turn out to be feeds redirect to the feed view, and a page that
declares its own feed offers it.
This commit is contained in:
parent
76d0906564
commit
3f71d324c9
@ -11,6 +11,7 @@ pub struct Page {
|
||||
pub url: String,
|
||||
pub html: String,
|
||||
pub content_type: String,
|
||||
pub status: u16,
|
||||
}
|
||||
|
||||
pub fn get(url: &str, limit: usize) -> Result<Page, String> {
|
||||
@ -22,6 +23,7 @@ pub fn get(url: &str, limit: usize) -> Result<Page, String> {
|
||||
.into();
|
||||
|
||||
let mut resp = agent.get(url).call().map_err(|e| format!("{url}: {e}"))?;
|
||||
let status = resp.status().as_u16();
|
||||
let final_url = resp.get_uri().to_string();
|
||||
let content_type = resp
|
||||
.headers()
|
||||
@ -37,7 +39,7 @@ pub fn get(url: &str, limit: usize) -> Result<Page, String> {
|
||||
.read_to_vec()
|
||||
.map_err(|e| format!("{url}: {e}"))?;
|
||||
|
||||
Ok(Page { url: final_url, html: decode(&bytes, &content_type), content_type })
|
||||
Ok(Page { url: final_url, html: decode(&bytes, &content_type), content_type, status })
|
||||
}
|
||||
|
||||
/// Decode to text using, in order: the Content-Type charset, a `<meta charset>`
|
||||
|
||||
@ -78,7 +78,7 @@ fn run() -> Result<(), String> {
|
||||
let page = if stdin {
|
||||
let mut buf = Vec::new();
|
||||
io::stdin().read_to_end(&mut buf).map_err(|e| e.to_string())?;
|
||||
fetch::Page { url: url.clone(), html: fetch::decode(&buf, ""), content_type: String::new() }
|
||||
fetch::Page { url: url.clone(), html: fetch::decode(&buf, ""), content_type: String::new(), status: 200 }
|
||||
} else {
|
||||
fetch::get(&url, LIMIT)?
|
||||
};
|
||||
|
||||
@ -12,6 +12,21 @@ pub struct Home {
|
||||
pub links: Vec<Entry>,
|
||||
#[serde(default, rename = "feed")]
|
||||
pub feeds: Vec<Entry>,
|
||||
#[serde(default)]
|
||||
pub search: Search,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct Search {
|
||||
/// Result-page URL with {q} where the encoded query goes. Any engine that
|
||||
/// serves plain HTML works; the parser falls back to reading links.
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
impl Default for Search {
|
||||
fn default() -> Self {
|
||||
Search { url: "https://html.duckduckgo.com/html/?q={q}".to_string() }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
@ -63,4 +78,11 @@ url = "https://lwn.net/headlines/newrss"
|
||||
[[feed]]
|
||||
name = "Phoronix"
|
||||
url = "https://www.phoronix.com/rss.php"
|
||||
|
||||
# Any engine that serves plain HTML. {q} is the encoded query.
|
||||
# DuckDuckGo rate-limits repeat visitors; a SearXNG instance does not.
|
||||
[search]
|
||||
url = "https://html.duckduckgo.com/html/?q={q}"
|
||||
# url = "https://searx.be/search?q={q}"
|
||||
# url = "https://search.marginalia.nu/search?query={q}"
|
||||
"#;
|
||||
|
||||
238
furst-serve/src/feed.rs
Normal file
238
furst-serve/src/feed.rs
Normal file
@ -0,0 +1,238 @@
|
||||
//! RSS and Atom, scanned rather than parsed.
|
||||
//!
|
||||
//! A feed only ever needs five fields per entry, and an HTML parser mangles
|
||||
//! XML, so this walks the tags directly. No XML dependency, no surprises.
|
||||
|
||||
pub struct Entry {
|
||||
pub title: String,
|
||||
pub link: String,
|
||||
pub date: String,
|
||||
pub summary: String,
|
||||
}
|
||||
|
||||
pub struct Feed {
|
||||
pub title: String,
|
||||
pub entries: Vec<Entry>,
|
||||
}
|
||||
|
||||
pub fn looks_like_feed(content_type: &str, body: &str) -> bool {
|
||||
let ct = content_type.to_ascii_lowercase();
|
||||
if ct.contains("rss") || ct.contains("atom") || ct.contains("xml") {
|
||||
return true;
|
||||
}
|
||||
let head = &body[..body.len().min(1024)].to_ascii_lowercase();
|
||||
head.contains("<rss") || head.contains("<feed") || head.contains("<rdf:rdf")
|
||||
}
|
||||
|
||||
pub fn parse(xml: &str) -> Option<Feed> {
|
||||
let mut entries = Vec::new();
|
||||
let mut cursor = 0;
|
||||
let mut first_item = xml.len();
|
||||
|
||||
while entries.len() < 200 {
|
||||
let item = ["item", "entry"]
|
||||
.iter()
|
||||
.filter_map(|name| block(xml, name, cursor))
|
||||
.min_by_key(|(start, _, _)| *start);
|
||||
let Some((start, end, block)) = item else { break };
|
||||
first_item = first_item.min(start);
|
||||
cursor = end;
|
||||
|
||||
let title = tag_text(block, "title").unwrap_or_default();
|
||||
let link = entry_link(block).unwrap_or_default();
|
||||
if title.is_empty() && link.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let date = ["pubDate", "published", "updated", "dc:date"]
|
||||
.iter()
|
||||
.find_map(|n| tag_text(block, n))
|
||||
.unwrap_or_default();
|
||||
let summary = ["description", "summary", "content"]
|
||||
.iter()
|
||||
.find_map(|n| tag_text(block, n))
|
||||
.map(|s| strip_tags(&s))
|
||||
.unwrap_or_default();
|
||||
|
||||
entries.push(Entry { title, link, date, summary });
|
||||
}
|
||||
|
||||
if entries.is_empty() {
|
||||
return None;
|
||||
}
|
||||
// The channel title is whichever <title> comes before the first entry.
|
||||
let title = tag_text(&xml[..first_item], "title").unwrap_or_default();
|
||||
Some(Feed { title, entries })
|
||||
}
|
||||
|
||||
/// Atom puts the URL in an attribute and may list several; RSS uses text.
|
||||
fn entry_link(block: &str) -> Option<String> {
|
||||
if let Some(text) = tag_text(block, "link").filter(|s| !s.is_empty()) {
|
||||
return Some(text);
|
||||
}
|
||||
let mut best: Option<String> = None;
|
||||
let mut cursor = 0;
|
||||
while let Some(i) = block[cursor..].find("<link") {
|
||||
let start = cursor + i;
|
||||
let end = block[start..].find('>').map(|e| start + e)?;
|
||||
let tag = &block[start..end];
|
||||
cursor = end;
|
||||
let Some(href) = attr(tag, "href") else { continue };
|
||||
let rel = attr(tag, "rel").unwrap_or_default();
|
||||
if rel.is_empty() || rel == "alternate" {
|
||||
return Some(href);
|
||||
}
|
||||
best.get_or_insert(href);
|
||||
}
|
||||
best
|
||||
}
|
||||
|
||||
/// Locate `<name ...> ... </name>`, returning its bounds and inner text.
|
||||
fn block<'a>(xml: &'a str, name: &str, from: usize) -> Option<(usize, usize, &'a str)> {
|
||||
let open = format!("<{name}");
|
||||
let close = format!("</{name}>");
|
||||
let mut cursor = from;
|
||||
loop {
|
||||
let start = cursor + xml[cursor..].find(&open)?;
|
||||
// `<item` must not match `<items`.
|
||||
let after = xml[start + open.len()..].chars().next();
|
||||
if after.is_some_and(|c| c.is_ascii_alphanumeric() || c == '-' || c == ':') {
|
||||
cursor = start + open.len();
|
||||
continue;
|
||||
}
|
||||
let body_start = start + xml[start..].find('>')? + 1;
|
||||
let end = body_start + xml[body_start..].find(&close)?;
|
||||
return Some((start, end + close.len(), &xml[body_start..end]));
|
||||
}
|
||||
}
|
||||
|
||||
fn tag_text(block: &str, name: &str) -> Option<String> {
|
||||
let (_, _, inner) = self::block(block, name, 0)?;
|
||||
Some(decode_entities(strip_cdata(inner).trim()))
|
||||
}
|
||||
|
||||
fn attr(tag: &str, name: &str) -> Option<String> {
|
||||
let key = format!("{name}=");
|
||||
let i = tag.find(&key)? + key.len();
|
||||
let rest = &tag[i..];
|
||||
let quote = rest.chars().next()?;
|
||||
if quote == '"' || quote == '\'' {
|
||||
let end = rest[1..].find(quote)? + 1;
|
||||
Some(decode_entities(&rest[1..end]))
|
||||
} else {
|
||||
let end = rest.find([' ', '>', '/']).unwrap_or(rest.len());
|
||||
Some(decode_entities(&rest[..end]))
|
||||
}
|
||||
}
|
||||
|
||||
fn strip_cdata(s: &str) -> &str {
|
||||
s.trim().strip_prefix("<![CDATA[").and_then(|r| r.strip_suffix("]]>")).unwrap_or(s)
|
||||
}
|
||||
|
||||
pub fn strip_tags(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
let mut in_tag = false;
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'<' => in_tag = true,
|
||||
'>' => in_tag = false,
|
||||
_ if !in_tag => out.push(c),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
decode_entities(&out).split_whitespace().collect::<Vec<_>>().join(" ")
|
||||
}
|
||||
|
||||
fn decode_entities(s: &str) -> String {
|
||||
if !s.contains('&') {
|
||||
return s.to_string();
|
||||
}
|
||||
let mut out = String::with_capacity(s.len());
|
||||
let mut rest = s;
|
||||
while let Some(i) = rest.find('&') {
|
||||
out.push_str(&rest[..i]);
|
||||
let tail = &rest[i..];
|
||||
let Some(semi) = tail[..tail.len().min(12)].find(';') else {
|
||||
out.push('&');
|
||||
rest = &tail[1..];
|
||||
continue;
|
||||
};
|
||||
let name = &tail[1..semi];
|
||||
let decoded = match name {
|
||||
"amp" => Some('&'),
|
||||
"lt" => Some('<'),
|
||||
"gt" => Some('>'),
|
||||
"quot" => Some('"'),
|
||||
"apos" => Some('\''),
|
||||
"nbsp" => Some(' '),
|
||||
n if n.starts_with("#x") || n.starts_with("#X") => {
|
||||
u32::from_str_radix(&n[2..], 16).ok().and_then(char::from_u32)
|
||||
}
|
||||
n if n.starts_with('#') => n[1..].parse().ok().and_then(char::from_u32),
|
||||
_ => None,
|
||||
};
|
||||
match decoded {
|
||||
Some(c) => out.push(c),
|
||||
None => out.push_str(&tail[..=semi]),
|
||||
}
|
||||
rest = &tail[semi + 1..];
|
||||
}
|
||||
out.push_str(rest);
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const RSS: &str = r#"<?xml version="1.0"?><rss><channel>
|
||||
<title>My Feed</title><link>https://ex.test/</link>
|
||||
<item><title>First & best</title><link>https://ex.test/1</link>
|
||||
<pubDate>Mon, 01 Jan 2026</pubDate>
|
||||
<description><![CDATA[<p>Hello <b>there</b></p>]]></description></item>
|
||||
<item><title>Second</title><link>https://ex.test/2</link></item>
|
||||
</channel></rss>"#;
|
||||
|
||||
const ATOM: &str = r#"<feed><title>Atom Feed</title>
|
||||
<link rel="self" href="https://ex.test/feed"/>
|
||||
<entry><title>Entry one</title>
|
||||
<link rel="self" href="https://ex.test/self"/>
|
||||
<link rel="alternate" href="https://ex.test/one"/>
|
||||
<updated>2026-01-01</updated><summary>A summary</summary></entry></feed>"#;
|
||||
|
||||
#[test]
|
||||
fn parses_rss_with_cdata_and_entities() {
|
||||
let f = parse(RSS).unwrap();
|
||||
assert_eq!(f.title, "My Feed");
|
||||
assert_eq!(f.entries.len(), 2);
|
||||
assert_eq!(f.entries[0].title, "First & best");
|
||||
assert_eq!(f.entries[0].link, "https://ex.test/1");
|
||||
assert_eq!(f.entries[0].summary, "Hello there");
|
||||
assert_eq!(f.entries[1].title, "Second");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn atom_prefers_the_alternate_link_over_self() {
|
||||
let f = parse(ATOM).unwrap();
|
||||
assert_eq!(f.title, "Atom Feed");
|
||||
assert_eq!(f.entries[0].link, "https://ex.test/one");
|
||||
assert_eq!(f.entries[0].summary, "A summary");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_feeds_by_type_or_shape() {
|
||||
assert!(looks_like_feed("application/rss+xml", ""));
|
||||
assert!(looks_like_feed("", "<?xml version=\"1.0\"?><rss>"));
|
||||
assert!(!looks_like_feed("text/html", "<!doctype html><html>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_document_with_no_items_is_not_a_feed() {
|
||||
assert!(parse("<html><body><p>hi</p></body></html>").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numeric_entities_decode() {
|
||||
assert_eq!(decode_entities("a'b — c"), "a'b — c");
|
||||
assert_eq!(decode_entities("100% ¬anentity"), "100% ¬anentity");
|
||||
}
|
||||
}
|
||||
@ -10,7 +10,7 @@ use scraper::Html;
|
||||
use crate::cache::Cache;
|
||||
use crate::config::Home;
|
||||
use crate::http::{Request, Response};
|
||||
use crate::page;
|
||||
use crate::{feed, page, search, sites};
|
||||
|
||||
const LIMIT: usize = 8 * 1024 * 1024;
|
||||
/// Below this much extracted text a page is a listing, not an article.
|
||||
@ -29,6 +29,8 @@ impl App {
|
||||
match req.path.as_str() {
|
||||
"/" => self.home(),
|
||||
"/read" => self.read(req),
|
||||
"/search" => self.search(req),
|
||||
"/feed" => self.feed(req),
|
||||
"/go" => self.go(req),
|
||||
_ => Response::status(404, page::error("Not found", "No such page.", None)),
|
||||
}
|
||||
@ -96,28 +98,39 @@ impl App {
|
||||
|
||||
let page = match fetch::get(url, LIMIT) {
|
||||
Ok(p) => p,
|
||||
Err(e) => {
|
||||
return Response::status(502, page::error("Could not fetch", &e, Some(url)));
|
||||
}
|
||||
Err(e) => return Response::status(502, page::error("Could not fetch", &e, Some(url))),
|
||||
};
|
||||
// A PDF or an image is not something to extract an article from.
|
||||
if !page.content_type.is_empty() && !page.content_type.contains("html") {
|
||||
if feed::looks_like_feed(&page.content_type, &page.html) {
|
||||
return Response::redirect(&format!("/feed?u={}", encode(&page.url)));
|
||||
}
|
||||
return Response::redirect(&page.url);
|
||||
}
|
||||
if feed::looks_like_feed(&page.content_type, &page.html) {
|
||||
return Response::redirect(&format!("/feed?u={}", encode(&page.url)));
|
||||
}
|
||||
|
||||
let (doc, base) = follow_refreshes(page);
|
||||
let article = extract::extract(&doc);
|
||||
let body = render::body(&doc, &article, &base, &render::Options {
|
||||
images: true,
|
||||
link_prefix: Some("/read?u=".to_string()),
|
||||
});
|
||||
|
||||
let html = self.article_page(&article, &base, &body);
|
||||
let html = self.render_page(&doc, &base);
|
||||
self.cache.put(&key, &html);
|
||||
Response::html(html)
|
||||
}
|
||||
|
||||
fn article_page(&self, article: &extract::Article, base: &str, body: &str) -> String {
|
||||
fn render_page(&self, doc: &Html, base: &str) -> String {
|
||||
// A site we know how to read structurally beats generic extraction.
|
||||
if let Some(view) = sites::adapt(doc, base) {
|
||||
let content = format!("<h1>{}</h1>\n<p class=\"meta\">{}</p>\n{}",
|
||||
page::escape(&view.title), page::escape(&host_of(base)), view.body);
|
||||
return page::shell(&view.title, "", &page::source_actions(base), &content);
|
||||
}
|
||||
|
||||
let article = extract::extract(doc);
|
||||
let body = render::body(doc, &article, base, &render::Options {
|
||||
images: true,
|
||||
link_prefix: Some("/read?u=".to_string()),
|
||||
});
|
||||
|
||||
let mut meta = Vec::new();
|
||||
if let Some(b) = &article.byline {
|
||||
meta.push(page::escape(b));
|
||||
@ -126,27 +139,141 @@ impl App {
|
||||
meta.push(page::escape(s));
|
||||
}
|
||||
meta.push(page::escape(&host_of(base)));
|
||||
if let Some(url) = sites::declared_feed(doc, base) {
|
||||
meta.push(format!("<a href=\"/feed?u={}\">feed</a>", encode(&url)));
|
||||
}
|
||||
|
||||
let title = if article.title.is_empty() { host_of(base) } else { article.title.clone() };
|
||||
let mut content = format!(
|
||||
"<h1>{}</h1>\n<p class=\"meta\">{}</p>\n",
|
||||
page::escape(&title),
|
||||
meta.join(" · ")
|
||||
);
|
||||
let mut content =
|
||||
format!("<h1>{}</h1>\n<p class=\"meta\">{}</p>\n", page::escape(&title), meta.join(" · "));
|
||||
|
||||
if text_len(body) < THIN {
|
||||
content.push_str(&page::notice(
|
||||
"",
|
||||
&format!(
|
||||
"Not much of an article here. Try the <a href=\"{}\">original</a> \
|
||||
or <a href=\"/go?u={}\">open it in the browser</a>.",
|
||||
page::escape(base),
|
||||
encode(base)
|
||||
),
|
||||
// Too little prose to be an article: show the page's structure instead.
|
||||
if text_len(&body) < THIN {
|
||||
let index = sites::index(doc, base);
|
||||
if index.is_empty() {
|
||||
content.push_str(&page::notice(
|
||||
"",
|
||||
&format!(
|
||||
"Not much here. Try the <a href=\"{}\">original</a> or \
|
||||
<a href=\"/go?u={}\">open it in the browser</a>.",
|
||||
page::escape(base),
|
||||
encode(base)
|
||||
),
|
||||
));
|
||||
content.push_str(&body);
|
||||
} else {
|
||||
content.push_str(&page::notice("", "No article on this page, so here are its links."));
|
||||
content.push_str(&index);
|
||||
}
|
||||
} else {
|
||||
content.push_str(&body);
|
||||
}
|
||||
page::shell(&title, "", &page::source_actions(base), &content)
|
||||
}
|
||||
|
||||
fn search(&self, req: &Request) -> Response {
|
||||
let Some(query) = req.param("q").filter(|q| !q.trim().is_empty()) else {
|
||||
return Response::redirect("/");
|
||||
};
|
||||
let key = format!("search:{query}");
|
||||
if let Some(hit) = self.cache.get(&key) {
|
||||
return Response::html(hit);
|
||||
}
|
||||
|
||||
let engine = Home::load().search.url;
|
||||
let hits = match search::search(&engine, query) {
|
||||
Ok(search::Outcome::Hits(h)) => h,
|
||||
Ok(search::Outcome::Blocked) => {
|
||||
let engine_host = host_of(&engine);
|
||||
return Response::status(502, page::error(
|
||||
"Search engine refused",
|
||||
&format!(
|
||||
"{engine_host} served a challenge page instead of results. \
|
||||
Point [search] url in the config at another HTML endpoint, \
|
||||
such as a SearXNG instance."
|
||||
),
|
||||
Some(&engine.replace("{q}", &encode(query))),
|
||||
));
|
||||
}
|
||||
Err(e) => return Response::status(502, page::error("Search failed", &e, None)),
|
||||
};
|
||||
|
||||
let mut body = format!("<h1>{}</h1>", page::escape(query));
|
||||
if hits.is_empty() {
|
||||
body.push_str(&page::notice("", "No results."));
|
||||
} else {
|
||||
body.push_str("<ul class=\"rows\">");
|
||||
for h in &hits {
|
||||
body.push_str(&format!(
|
||||
"<li><a class=\"t\" href=\"{}\">{}</a><span class=\"u\">{}</span>{}</li>",
|
||||
page::read_link(&h.url),
|
||||
page::escape(&h.title),
|
||||
page::escape(&host_of(&h.url)),
|
||||
if h.snippet.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("<span class=\"s\">{}</span>", page::escape(&h.snippet))
|
||||
}
|
||||
));
|
||||
}
|
||||
body.push_str("</ul>");
|
||||
}
|
||||
|
||||
let html = page::shell(&format!("{query} — search"), query, "", &body);
|
||||
if !hits.is_empty() {
|
||||
self.cache.put(&key, &html);
|
||||
}
|
||||
Response::html(html)
|
||||
}
|
||||
|
||||
fn feed(&self, req: &Request) -> Response {
|
||||
let Some(url) = req.param("u").filter(|u| !u.is_empty()) else {
|
||||
return Response::redirect("/");
|
||||
};
|
||||
let key = format!("feed:{url}");
|
||||
if req.param("fresh").is_none()
|
||||
&& let Some(hit) = self.cache.get(&key)
|
||||
{
|
||||
return Response::html(hit);
|
||||
}
|
||||
|
||||
let page = match fetch::get(url, LIMIT) {
|
||||
Ok(p) => p,
|
||||
Err(e) => return Response::status(502, page::error("Could not fetch", &e, Some(url))),
|
||||
};
|
||||
let Some(parsed) = feed::parse(&page.html) else {
|
||||
return Response::status(
|
||||
502,
|
||||
page::error("Not a feed", "No RSS items or Atom entries in that document.", Some(url)),
|
||||
);
|
||||
};
|
||||
|
||||
let title = if parsed.title.is_empty() { host_of(url) } else { parsed.title.clone() };
|
||||
let mut body = format!(
|
||||
"<h1>{}</h1><p class=\"meta\">{} · {} entries</p><ul class=\"rows\">",
|
||||
page::escape(&title),
|
||||
page::escape(&host_of(url)),
|
||||
parsed.entries.len()
|
||||
);
|
||||
for e in &parsed.entries {
|
||||
let label = if e.title.is_empty() { host_of(&e.link) } else { e.title.clone() };
|
||||
body.push_str(&format!(
|
||||
"<li><a class=\"t\" href=\"{}\">{}</a><span class=\"u\">{}</span>{}</li>",
|
||||
page::read_link(&e.link),
|
||||
page::escape(&label),
|
||||
page::escape(&e.date),
|
||||
if e.summary.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("<span class=\"s\">{}</span>", page::escape(&truncate(&e.summary, 280)))
|
||||
}
|
||||
));
|
||||
}
|
||||
content.push_str(body);
|
||||
page::shell(&title, "", &page::source_actions(base), &content)
|
||||
body.push_str("</ul>");
|
||||
|
||||
let html = page::shell(&title, "", &page::source_actions(url), &body);
|
||||
self.cache.put(&key, &html);
|
||||
Response::html(html)
|
||||
}
|
||||
|
||||
fn go(&self, req: &Request) -> Response {
|
||||
@ -215,6 +342,15 @@ pub fn text_len(html: &str) -> usize {
|
||||
count
|
||||
}
|
||||
|
||||
fn truncate(s: &str, max: usize) -> String {
|
||||
if s.chars().count() <= max {
|
||||
return s.to_string();
|
||||
}
|
||||
let cut: String = s.chars().take(max).collect();
|
||||
let at = cut.rfind(' ').unwrap_or(cut.len());
|
||||
format!("{}…", &cut[..at])
|
||||
}
|
||||
|
||||
pub fn host_of(url: &str) -> String {
|
||||
let rest = url.split_once("://").map(|(_, r)| r).unwrap_or(url);
|
||||
let authority = rest.split(['/', '?', '#']).next().unwrap_or(rest);
|
||||
|
||||
@ -1,8 +1,11 @@
|
||||
mod cache;
|
||||
mod config;
|
||||
mod feed;
|
||||
mod handler;
|
||||
mod http;
|
||||
mod page;
|
||||
mod search;
|
||||
mod sites;
|
||||
|
||||
use std::env;
|
||||
use std::io::Write;
|
||||
|
||||
238
furst-serve/src/search.rs
Normal file
238
furst-serve/src/search.rs
Normal file
@ -0,0 +1,238 @@
|
||||
//! Search, rendered as our own list. A results page costs a few kilobytes
|
||||
//! instead of a megabyte.
|
||||
//!
|
||||
//! The engine is configurable because every free HTML endpoint eventually
|
||||
//! rate-limits a repeat visitor, and being told so beats an empty page.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use furst_read::urljoin::{decode, encode, join};
|
||||
use furst_read::{extract, fetch};
|
||||
use scraper::{ElementRef, Html, Selector};
|
||||
|
||||
const LIMIT: usize = 2 * 1024 * 1024;
|
||||
|
||||
pub struct Hit {
|
||||
pub title: String,
|
||||
pub url: String,
|
||||
pub snippet: String,
|
||||
}
|
||||
|
||||
pub enum Outcome {
|
||||
Hits(Vec<Hit>),
|
||||
/// The engine served a challenge or an anomaly page instead of results.
|
||||
Blocked,
|
||||
}
|
||||
|
||||
pub fn search(engine: &str, query: &str) -> Result<Outcome, String> {
|
||||
let url = if engine.contains("{q}") {
|
||||
engine.replace("{q}", &encode(query))
|
||||
} else {
|
||||
format!("{engine}{}", encode(query))
|
||||
};
|
||||
let page = fetch::get(&url, LIMIT)?;
|
||||
let doc = Html::parse_document(&page.html);
|
||||
let hits = hits(&doc, &page.url);
|
||||
|
||||
if hits.is_empty() && blocked(page.status, &page.html) {
|
||||
return Ok(Outcome::Blocked);
|
||||
}
|
||||
Ok(Outcome::Hits(hits))
|
||||
}
|
||||
|
||||
/// Engines answer a challenge with 2xx and a page that has no results on it,
|
||||
/// so the body has to be inspected rather than the status alone.
|
||||
fn blocked(status: u16, body: &str) -> bool {
|
||||
if status != 200 {
|
||||
return true;
|
||||
}
|
||||
let head = body[..body.len().min(20_000)].to_ascii_lowercase();
|
||||
[
|
||||
"anomaly",
|
||||
"challenge",
|
||||
"captcha",
|
||||
"unusual traffic",
|
||||
"are you a robot",
|
||||
"verifying your browser",
|
||||
"just a moment",
|
||||
"enable javascript and cookies",
|
||||
]
|
||||
.iter()
|
||||
.any(|m| head.contains(m))
|
||||
}
|
||||
|
||||
static DDG: LazyLock<Selector> = LazyLock::new(|| Selector::parse("a.result__a").unwrap());
|
||||
static DDG_SNIP: LazyLock<Selector> = LazyLock::new(|| Selector::parse(".result__snippet").unwrap());
|
||||
static SEARX: LazyLock<Selector> = LazyLock::new(|| Selector::parse("article.result h3 a, .result h3 a").unwrap());
|
||||
static SEARX_SNIP: LazyLock<Selector> = LazyLock::new(|| Selector::parse("p.content, .content").unwrap());
|
||||
static RESULT: LazyLock<Selector> = LazyLock::new(|| Selector::parse("div.result, article.result, .result").unwrap());
|
||||
static ANY_LINK: LazyLock<Selector> = LazyLock::new(|| Selector::parse("a[href]").unwrap());
|
||||
static HEADING_LINK: LazyLock<Selector> =
|
||||
LazyLock::new(|| Selector::parse("h1 a[href], h2 a[href], h3 a[href]").unwrap());
|
||||
|
||||
fn hits(doc: &Html, base: &str) -> Vec<Hit> {
|
||||
let structured = by_result_blocks(doc, base);
|
||||
if !structured.is_empty() {
|
||||
return structured;
|
||||
}
|
||||
// Unknown engine: reading the links off the page still works.
|
||||
generic(doc, base)
|
||||
}
|
||||
|
||||
fn by_result_blocks(doc: &Html, base: &str) -> Vec<Hit> {
|
||||
let mut out = Vec::new();
|
||||
for result in doc.select(&RESULT) {
|
||||
if result.value().attr("class").is_some_and(|c| c.contains("result--ad")) {
|
||||
continue;
|
||||
}
|
||||
let Some(link) = result.select(&DDG).next().or_else(|| result.select(&SEARX).next()) else {
|
||||
continue;
|
||||
};
|
||||
let url = target(link.value().attr("href").unwrap_or(""), base);
|
||||
if url.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let snippet = result
|
||||
.select(&DDG_SNIP)
|
||||
.next()
|
||||
.or_else(|| result.select(&SEARX_SNIP).next())
|
||||
.map(text_of)
|
||||
.unwrap_or_default();
|
||||
out.push(Hit { title: text_of(link), url, snippet });
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn generic(doc: &Html, base: &str) -> Vec<Hit> {
|
||||
// Result titles are headings on nearly every engine; footer chrome is not.
|
||||
let headings = collect(doc.select(&HEADING_LINK), base);
|
||||
if headings.len() >= 3 {
|
||||
return headings;
|
||||
}
|
||||
let all = collect(doc.select(&ANY_LINK), base);
|
||||
if all.len() > headings.len() { all } else { headings }
|
||||
}
|
||||
|
||||
fn collect<'a>(links: impl Iterator<Item = ElementRef<'a>>, base: &str) -> Vec<Hit> {
|
||||
let engine_host = crate::handler::host_of(base);
|
||||
let mut out: Vec<Hit> = Vec::new();
|
||||
for link in links {
|
||||
let title = text_of(link);
|
||||
if title.chars().count() < 15 {
|
||||
continue;
|
||||
}
|
||||
let url = target(link.value().attr("href").unwrap_or(""), base);
|
||||
// Skip the engine's own navigation.
|
||||
if url.is_empty() || crate::handler::host_of(&url) == engine_host {
|
||||
continue;
|
||||
}
|
||||
if out.iter().any(|h| h.url == url) {
|
||||
continue;
|
||||
}
|
||||
out.push(Hit { title, url, snippet: String::new() });
|
||||
if out.len() >= 40 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Resolve a result href, unwrapping the redirect most engines wrap them in.
|
||||
fn target(href: &str, base: &str) -> String {
|
||||
let href = href.trim();
|
||||
if href.is_empty() || href.starts_with('#') {
|
||||
return String::new();
|
||||
}
|
||||
for key in ["uddg=", "url=", "u3=", "q="] {
|
||||
if let Some(i) = href.find(key) {
|
||||
let rest = &href[i + key.len()..];
|
||||
let end = rest.find('&').unwrap_or(rest.len());
|
||||
let decoded = decode(&rest[..end]);
|
||||
if decoded.starts_with("http") {
|
||||
return decoded;
|
||||
}
|
||||
}
|
||||
}
|
||||
let abs = join(base, href);
|
||||
if abs.starts_with("http") { abs } else { String::new() }
|
||||
}
|
||||
|
||||
fn text_of(el: ElementRef) -> String {
|
||||
extract::normalize(&extract::inner_text(el))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn unwraps_the_redirect_wrapper() {
|
||||
let b = "https://html.duckduckgo.com/html/";
|
||||
assert_eq!(
|
||||
target("//duckduckgo.com/l/?uddg=https%3A%2F%2Fex.test%2Fa&rut=abc", b),
|
||||
"https://ex.test/a"
|
||||
);
|
||||
assert_eq!(target("https://ex.test/a", b), "https://ex.test/a");
|
||||
assert_eq!(target("#", b), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_duckduckgo_results_and_skips_ads() {
|
||||
let doc = Html::parse_document(
|
||||
r#"<div class="result result--ad"><a class="result__a" href="https://ad.test">Ad</a></div>
|
||||
<div class="result"><a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fex.test%2Fa">Title</a>
|
||||
<a class="result__snippet">A snippet</a></div>"#,
|
||||
);
|
||||
let h = hits(&doc, "https://html.duckduckgo.com/html/");
|
||||
assert_eq!(h.len(), 1);
|
||||
assert_eq!(h[0].title, "Title");
|
||||
assert_eq!(h[0].url, "https://ex.test/a");
|
||||
assert_eq!(h[0].snippet, "A snippet");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_searxng_results() {
|
||||
let doc = Html::parse_document(
|
||||
r#"<article class="result"><h3><a href="https://ex.test/b">Second thing</a></h3>
|
||||
<p class="content">Some content</p></article>"#,
|
||||
);
|
||||
let h = hits(&doc, "https://searx.test/search");
|
||||
assert_eq!(h.len(), 1);
|
||||
assert_eq!(h[0].url, "https://ex.test/b");
|
||||
assert_eq!(h[0].snippet, "Some content");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generic_prefers_heading_links_over_footer_chrome() {
|
||||
let doc = Html::parse_document(
|
||||
r#"<h2><a href="https://a.test/1">First result title here</a></h2>
|
||||
<h2><a href="https://b.test/2">Second result title here</a></h2>
|
||||
<h2><a href="https://c.test/3">Third result title here</a></h2>
|
||||
<footer><a href="https://engine-blog.test/x">About the crawler and its IPs</a></footer>"#,
|
||||
);
|
||||
let h = hits(&doc, "https://engine.test/search");
|
||||
assert_eq!(h.len(), 3);
|
||||
assert!(h.iter().all(|x| !x.url.contains("engine-blog")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn falls_back_to_links_for_an_unknown_engine() {
|
||||
let doc = Html::parse_document(
|
||||
r#"<a href="/settings">Prefs</a>
|
||||
<a href="https://engine.test/about">About this engine</a>
|
||||
<a href="https://ex.test/c">A result title long enough</a>"#,
|
||||
);
|
||||
let h = hits(&doc, "https://engine.test/search");
|
||||
assert_eq!(h.len(), 1, "engine's own pages and short links are skipped");
|
||||
assert_eq!(h[0].url, "https://ex.test/c");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_challenge_page_is_recognised() {
|
||||
assert!(blocked(202, ""));
|
||||
assert!(blocked(200, "<html><body>anomaly detected</body></html>"));
|
||||
// What searx.be actually serves when it does not want to answer.
|
||||
assert!(blocked(200, "<title>Verifying your browser\u{2026}</title>"));
|
||||
assert!(!blocked(200, "<html><body>normal results</body></html>"));
|
||||
}
|
||||
}
|
||||
207
furst-serve/src/sites.rs
Normal file
207
furst-serve/src/sites.rs
Normal file
@ -0,0 +1,207 @@
|
||||
//! Views for pages that are not articles.
|
||||
//!
|
||||
//! Extraction looks for prose, so a front page or a comment thread correctly
|
||||
//! yields almost nothing. These render the structure instead.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use furst_read::urljoin::join;
|
||||
use scraper::{ElementRef, Html, Selector};
|
||||
|
||||
use crate::page::{escape, read_link};
|
||||
|
||||
pub struct View {
|
||||
pub title: String,
|
||||
pub body: String,
|
||||
}
|
||||
|
||||
/// A site-specific view, when one applies.
|
||||
pub fn adapt(doc: &Html, base: &str) -> Option<View> {
|
||||
if base.contains("news.ycombinator.com") {
|
||||
return Some(hacker_news(doc, base));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Hacker News
|
||||
|
||||
static HN_ROW: LazyLock<Selector> = LazyLock::new(|| Selector::parse("tr.athing.submission").unwrap());
|
||||
static HN_TITLELINE: LazyLock<Selector> = LazyLock::new(|| Selector::parse("span.titleline > a").unwrap());
|
||||
static HN_COMMENT: LazyLock<Selector> = LazyLock::new(|| Selector::parse("tr.athing.comtr").unwrap());
|
||||
static HN_IND: LazyLock<Selector> = LazyLock::new(|| Selector::parse("td.ind").unwrap());
|
||||
static HN_USER: LazyLock<Selector> = LazyLock::new(|| Selector::parse("a.hnuser").unwrap());
|
||||
static HN_AGE: LazyLock<Selector> = LazyLock::new(|| Selector::parse("span.age > a").unwrap());
|
||||
static HN_TEXT: LazyLock<Selector> = LazyLock::new(|| Selector::parse(".commtext").unwrap());
|
||||
static HN_SUBTEXT: LazyLock<Selector> = LazyLock::new(|| Selector::parse("td.subtext").unwrap());
|
||||
|
||||
fn hacker_news(doc: &Html, base: &str) -> View {
|
||||
let stories = stories(doc, base);
|
||||
let comments = comments(doc, base);
|
||||
|
||||
let mut body = String::new();
|
||||
if !stories.is_empty() {
|
||||
body.push_str(&stories);
|
||||
}
|
||||
if !comments.is_empty() {
|
||||
if !stories.is_empty() {
|
||||
body.push_str("<p class=\"sec\">Comments</p>");
|
||||
}
|
||||
body.push_str(&comments);
|
||||
}
|
||||
if body.is_empty() {
|
||||
body.push_str("<p class=\"notice\">Nothing to show on this page.</p>");
|
||||
}
|
||||
View { title: "Hacker News".to_string(), body }
|
||||
}
|
||||
|
||||
fn stories(doc: &Html, base: &str) -> String {
|
||||
let mut out = String::new();
|
||||
for row in doc.select(&HN_ROW) {
|
||||
let Some(link) = row.select(&HN_TITLELINE).next() else { continue };
|
||||
let title = text_of(link);
|
||||
let target = join(base, link.value().attr("href").unwrap_or(""));
|
||||
|
||||
// The score and comment count live in the row after the title.
|
||||
let sub = row
|
||||
.next_siblings()
|
||||
.filter_map(ElementRef::wrap)
|
||||
.find_map(|s| s.select(&HN_SUBTEXT).next());
|
||||
let mut meta = Vec::new();
|
||||
let mut discussion = None;
|
||||
if let Some(sub) = sub {
|
||||
let score = sub
|
||||
.select(&Selector::parse("span.score").unwrap())
|
||||
.next()
|
||||
.map(text_of)
|
||||
.unwrap_or_default();
|
||||
if !score.is_empty() {
|
||||
meta.push(escape(&score));
|
||||
}
|
||||
if let Some(user) = sub.select(&HN_USER).next() {
|
||||
meta.push(escape(&text_of(user)));
|
||||
}
|
||||
// The discussion link is the last one, reading "N comments".
|
||||
for a in sub.select(&Selector::parse("a").unwrap()) {
|
||||
let t = text_of(a);
|
||||
if t.contains("comment") || t == "discuss" {
|
||||
discussion = Some((join(base, a.value().attr("href").unwrap_or("")), t));
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some((_, label)) = &discussion {
|
||||
meta.push(escape(label));
|
||||
}
|
||||
|
||||
out.push_str(&format!(
|
||||
"<li><a class=\"t\" href=\"{}\">{}</a><span class=\"u\">{}{}</span></li>",
|
||||
read_link(&target),
|
||||
escape(&title),
|
||||
escape(&crate::handler::host_of(&target)),
|
||||
if meta.is_empty() { String::new() } else { format!(" · {}", meta.join(" · ")) }
|
||||
));
|
||||
if let Some((url, label)) = discussion {
|
||||
out.push_str(&format!(
|
||||
"<li style=\"margin-top:-0.9rem\"><a class=\"u\" href=\"{}\">→ {}</a></li>",
|
||||
read_link(&url),
|
||||
escape(&label)
|
||||
));
|
||||
}
|
||||
}
|
||||
if out.is_empty() { out } else { format!("<ul class=\"rows\">{out}</ul>") }
|
||||
}
|
||||
|
||||
fn comments(doc: &Html, base: &str) -> String {
|
||||
let mut out = String::new();
|
||||
for row in doc.select(&HN_COMMENT) {
|
||||
let depth: usize = row
|
||||
.select(&HN_IND)
|
||||
.next()
|
||||
.and_then(|td| td.value().attr("indent"))
|
||||
.and_then(|d| d.parse().ok())
|
||||
.unwrap_or(0);
|
||||
let user = row.select(&HN_USER).next().map(text_of).unwrap_or_default();
|
||||
let age = row.select(&HN_AGE).next().map(text_of).unwrap_or_default();
|
||||
let Some(text) = row.select(&HN_TEXT).next() else { continue };
|
||||
|
||||
// Reuse the article renderer so links inside comments route back
|
||||
// through the reader like every other link does.
|
||||
let body = furst_read::render::body(
|
||||
doc,
|
||||
&furst_read::extract::Article {
|
||||
title: String::new(),
|
||||
byline: None,
|
||||
site: None,
|
||||
nodes: vec![text.id()],
|
||||
},
|
||||
base,
|
||||
&furst_read::render::Options { images: false, link_prefix: Some("/read?u=".to_string()) },
|
||||
);
|
||||
out.push_str(&format!(
|
||||
"<div class=\"cmt\" style=\"margin-left:{}rem\"><div class=\"h\">{} · {}</div>{}</div>",
|
||||
(depth.min(12) as f32) * 1.1,
|
||||
escape(&user),
|
||||
escape(&age),
|
||||
body
|
||||
));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------- generic index
|
||||
|
||||
static LINKS: LazyLock<Selector> = LazyLock::new(|| Selector::parse("a[href]").unwrap());
|
||||
|
||||
/// The fallback for a page with no article: every link on it worth following.
|
||||
pub fn index(doc: &Html, base: &str) -> String {
|
||||
let mut seen: Vec<String> = Vec::new();
|
||||
let mut rows = String::new();
|
||||
|
||||
for a in doc.select(&LINKS) {
|
||||
let href = a.value().attr("href").unwrap_or("").trim();
|
||||
if href.is_empty() || href.starts_with('#') || href.to_ascii_lowercase().starts_with("javascript:") {
|
||||
continue;
|
||||
}
|
||||
let text = text_of(a);
|
||||
// Navigation is short; things worth reading are not.
|
||||
if text.chars().count() < 18 {
|
||||
continue;
|
||||
}
|
||||
let target = join(base, href);
|
||||
if !target.starts_with("http") || target == base || seen.contains(&target) {
|
||||
continue;
|
||||
}
|
||||
seen.push(target.clone());
|
||||
rows.push_str(&format!(
|
||||
"<li><a class=\"t\" href=\"{}\">{}</a><span class=\"u\">{}</span></li>",
|
||||
read_link(&target),
|
||||
escape(&text),
|
||||
escape(&crate::handler::host_of(&target))
|
||||
));
|
||||
if seen.len() >= 200 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if rows.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("<p class=\"sec\">{} links on this page</p><ul class=\"rows\">{rows}</ul>", seen.len())
|
||||
}
|
||||
}
|
||||
|
||||
/// A feed the page declares for itself, so the reader can offer it.
|
||||
pub fn declared_feed(doc: &Html, base: &str) -> Option<String> {
|
||||
static ALT: LazyLock<Selector> =
|
||||
LazyLock::new(|| Selector::parse("link[rel~=\"alternate\"][href]").unwrap());
|
||||
for link in doc.select(&ALT) {
|
||||
let kind = link.value().attr("type").unwrap_or("");
|
||||
if kind.contains("rss") || kind.contains("atom") {
|
||||
return Some(join(base, link.value().attr("href").unwrap_or("")));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn text_of(el: ElementRef) -> String {
|
||||
furst_read::extract::normalize(&furst_read::extract::inner_text(el))
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user