use std::fs::{self, File}; use std::io::Cursor; use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Duration; use anyhow::{Context, Result, bail}; use epub_builder::{EpubBuilder, EpubContent, ReferenceType, ZipLibrary}; use legible::parse; use reqwest::blocking::Client; use reqwest::header::{ACCEPT, CONTENT_TYPE, USER_AGENT}; use scraper::{Html, Selector}; use url::Url; // browser user-agent value used for the https requests const USER_AGENT_VALUE: &str = concat!("link2epub/", env!("CARGO_PKG_VERSION")); static TEMPORARY_IMAGE_ID: AtomicU64 = AtomicU64::new(0); const STYLESHEET: &str = r#" body { font-family: serif; line-height: 1.5; margin: 5%; } h1 { line-height: 1.15; margin-bottom: 0.3em; } .byline, .source { color: #555; font-size: 0.9em; } .article p { margin: 0 0 1em; } .article blockquote { border-left: 0.2em solid #999; margin-left: 0; padding-left: 1em; } .article pre { white-space: pre-wrap; } figure { margin: 1.5em 0; text-align: center; } img { height: auto; max-width: 100%; } "#; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum Format { Epub, Mobi, } pub struct Options { pub url: String, pub format: Format, } #[derive(Debug)] struct Article { title: String, author: Option, content_html: String, source_url: String, images: Vec, image_urls: Vec<(Url, String)>, } #[derive(Debug)] struct ArticleImage { source_url: Url, path: String, media_type: &'static str, bytes: Vec, } /// download an article and write it in the requested ebook format pub fn convert(options: Options) -> Result { let url = validate_url(&options.url)?; eprintln!("downloading article: {url}"); let article = download_article(&url)?; let output = default_output_path(&article.title, options.format); eprintln!("article: {}", article.title); eprintln!("output: {}", output.display()); match options.format { Format::Epub => write_epub(&article, &output)?, Format::Mobi => write_mobi(&article, &output)?, } Ok(output) } fn validate_url(input: &str) -> Result { let url = Url::parse(input).context("invalid article URL")?; if !matches!(url.scheme(), "http" | "https") { bail!("url scheme must be http or https"); } Ok(url) } /// fetches the page and applies a reader-mode extraction algorithm fn download_article(url: &Url) -> Result
{ let client = Client::builder() .timeout(Duration::from_secs(30)) .redirect(reqwest::redirect::Policy::limited(10)) .build() .context("could not initialize http client")?; let response = client .get(url.clone()) .header(USER_AGENT, USER_AGENT_VALUE) .header(ACCEPT, "text/html,application/xhtml+xml") .send() .with_context(|| format!("could not download {url}"))? .error_for_status() .with_context(|| format!("server returned an error for {url}"))?; let final_url = response.url().clone(); let html = response .text() .context("could not read the downloaded page")?; if final_url != *url { eprintln!("redirected to: {final_url}"); } eprintln!("extracting reader-mode content..."); let mut article = extract_article(&html, &final_url)?; let image_count = article.image_urls.len(); eprintln!( "found {} article image{}.", image_count, if image_count == 1 { "" } else { "s" } ); article.images = download_images(&client, &article.image_urls); article.image_urls.clear(); if image_count > 0 { eprintln!("embedded {} of {image_count} images.", article.images.len()); } Ok(article) } fn extract_article(html: &str, url: &Url) -> Result
{ let extracted = parse(html, Some(url.as_str()), None) .context("could not identify the article content on this page")?; let title = extracted.title.trim().to_owned(); if title.is_empty() { bail!("the page did not contain an article title"); } let image_urls = discover_image_urls(&extracted.content, url); let text = html2text::from_read(extracted.content.as_bytes(), 100) .context("could not convert article HTML to readable text")?; if text.trim().is_empty() { bail!("the page did not contain readable article text"); } Ok(Article { title, author: extracted.byline.filter(|value| !value.trim().is_empty()), content_html: extracted.content, source_url: url.to_string(), images: Vec::new(), image_urls, }) } fn discover_image_urls(html: &str, base_url: &Url) -> Vec<(Url, String)> { let document = Html::parse_fragment(html); let selector = Selector::parse("img").expect("valid image selector"); document .select(&selector) .filter_map(|image| { let attributes = image.value(); let source = attributes .attr("data-src") .or_else(|| attributes.attr("data-lazy-src")) .or_else(|| attributes.attr("src"))?; let url = base_url.join(source).ok()?; matches!(url.scheme(), "http" | "https") .then(|| (url, attributes.attr("alt").unwrap_or_default().to_owned())) }) .collect() } /// downloads supported article images (treats individual broken images as optional) fn download_images(client: &Client, candidates: &[(Url, String)]) -> Vec { let mut images = Vec::new(); for (index, (url, _alt)) in candidates.iter().enumerate() { eprintln!( "[{}/{}] downloading image: {url}", index + 1, candidates.len() ); let result = (|| -> Result { let response = client .get(url.clone()) .header(USER_AGENT, USER_AGENT_VALUE) .send()? .error_for_status()?; let reported_type = response .headers() .get(CONTENT_TYPE) .and_then(|value| value.to_str().ok()) .map(|value| { value .split(';') .next() .unwrap_or(value) .trim() .to_ascii_lowercase() }); let bytes = response.bytes()?.to_vec(); if bytes.is_empty() { bail!("empty image"); } let source_type = reported_type .as_deref() .and_then(supported_image_type) .or_else(|| image_type_from_url(url).and_then(supported_image_type)) .context("unsupported image type")?; let (bytes, media_type) = if source_type == "image/avif" { eprintln!( "[{}/{}] converting avif to png...", index + 1, candidates.len() ); ( avif_to_png(&bytes).context("could not convert avif image to png")?, "image/png", ) } else { (bytes, source_type) }; let extension = image_extension(media_type); Ok(ArticleImage { source_url: url.clone(), path: format!("images/image-{}.{}", images.len() + 1, extension), media_type, bytes, }) })(); match result { Ok(image) => { eprintln!("[{}/{}] image ready.", index + 1, candidates.len()); images.push(image); } Err(error) => eprintln!("warning: skipping image {url}: {error:#}"), } } images } /// transcodes avif to the broadly supported png format used inside the ebook. fn avif_to_png(bytes: &[u8]) -> Result> { match avif_to_png_rust(bytes) { Ok(png) => Ok(png), Err(rust_error) => { eprintln!("pure-rust avif decoder could not handle this image; trying avifdec..."); avif_to_png_with_avifdec(bytes).with_context(|| { format!("pure-rust decoder failed ({rust_error}); avifdec fallback also failed") }) } } } fn avif_to_png_rust(bytes: &[u8]) -> Result> { let image = avif_rust::image_from_bytes(bytes).context("could not decode avif pixels")?; lodepng::encode_memory( &image.rgba, image.width, image.height, lodepng::ColorType::RGBA, 8, ) .context("could not encode png pixels") } /// uses libavif's reference CLI for bitstreams unsupported by the Rust decoder. fn avif_to_png_with_avifdec(bytes: &[u8]) -> Result> { let id = TEMPORARY_IMAGE_ID.fetch_add(1, Ordering::Relaxed); let stem = format!("link2epub-{}-{id}", std::process::id()); let input = std::env::temp_dir().join(format!("{stem}.avif")); let output = std::env::temp_dir().join(format!("{stem}.png")); fs::write(&input, bytes).context("could not create temporary avif")?; let result = Command::new("avifdec").arg(&input).arg(&output).output(); let _ = fs::remove_file(&input); let command_output = result.context("avifdec is not available on PATH")?; if !command_output.status.success() { let _ = fs::remove_file(&output); let stderr = String::from_utf8_lossy(&command_output.stderr); bail!("avifdec could not transcode the image: {}", stderr.trim()); } let png = fs::read(&output).context("avifdec did not create the temporary png")?; let _ = fs::remove_file(&output); if !png.starts_with(b"\x89png\r\n\x1a\n") { bail!("avifdec produced an invalid png"); } Ok(png) } fn supported_image_type(value: &str) -> Option<&'static str> { match value.trim().to_ascii_lowercase().as_str() { "image/jpeg" | "image/jpg" => Some("image/jpeg"), "image/png" => Some("image/png"), "image/gif" => Some("image/gif"), "image/webp" => Some("image/webp"), "image/svg+xml" => Some("image/svg+xml"), "image/avif" => Some("image/avif"), _ => None, } } fn image_type_from_url(url: &Url) -> Option<&'static str> { let path = url.path().to_ascii_lowercase(); let extension = path.rsplit('.').next()?; match extension { "jpg" | "jpeg" => Some("image/jpeg"), "png" => Some("image/png"), "gif" => Some("image/gif"), "webp" => Some("image/webp"), "svg" => Some("image/svg+xml"), "avif" => Some("image/avif"), _ => None, } } fn image_extension(media_type: &str) -> &'static str { match media_type { "image/jpeg" => "jpg", "image/png" => "png", "image/gif" => "gif", "image/webp" => "webp", "image/svg+xml" => "svg", "image/avif" => "avif", _ => unreachable!("only supported image types reach this point"), } } fn write_epub(article: &Article, path: &Path) -> Result<()> { eprintln!("building epub: {}", path.display()); let file = File::create(path).with_context(|| format!("could not create {}", path.display()))?; build_epub(article, file).with_context(|| format!("could not generate {}", path.display())) } fn build_epub(article: &Article, output: impl std::io::Write + std::io::Seek) -> Result<()> { let author = article.author.as_deref().unwrap_or("unknown author"); let content = article_xhtml(article); let mut builder = EpubBuilder::new(ZipLibrary::new()?)?; builder .metadata("title", &article.title)? .metadata("author", author)? .metadata("lang", "en")? .stylesheet(STYLESHEET.as_bytes())?; for image in &article.images { builder.add_resource( &image.path, Cursor::new(image.bytes.as_slice()), image.media_type, )?; } builder.add_content( EpubContent::new("article.xhtml", Cursor::new(content.into_bytes())) .title(&article.title) .reftype(ReferenceType::Text), )?; builder.generate(output)?; Ok(()) } /// builds an intermediate epub and delegates mobi encoding to Calibre. fn write_mobi(article: &Article, output: &Path) -> Result<()> { let temporary = temporary_epub_path(output); eprintln!("preparing intermediate epub for mobi conversion..."); write_epub(article, &temporary)?; let conversion = if command_available("ebook-convert") { eprintln!("converting epub to mobi with Calibre..."); Command::new("ebook-convert") .arg(&temporary) .arg(output) .status() .context("could not run ebook-convert") } else { let _ = fs::remove_file(&temporary); bail!( "mobi output requires Calibre's ebook-convert on PATH; install Calibre or use --format epub" ); }; let status = conversion?; let _ = fs::remove_file(&temporary); if !status.success() { bail!("mobi converter exited with status {status}"); } if !output.is_file() { bail!( "mobi converter succeeded but did not create {}", output.display() ); } Ok(()) } fn command_available(command: &str) -> bool { Command::new(command) .arg("--version") .output() .is_ok_and(|output| output.status.success()) } fn article_xhtml(article: &Article) -> String { let byline = article .author .as_deref() .map_or_else(String::new, |author| { format!("

{}

", escape_html(author)) }); let body = render_article_html(article); format!( r#" {title}

{title}

{byline}

Source: {url}

{body}
"#, title = escape_html(&article.title), url = escape_html(&article.source_url), ) } /// Converts reader-mode HTML to a conservative XHTML subset and rewrites images locally. fn render_article_html(article: &Article) -> String { let document = Html::parse_fragment(&article.content_html); let base_url = Url::parse(&article.source_url).expect("stored source url is valid"); let mut output = String::new(); for child in document.tree.root().children() { render_node(child, &base_url, &article.images, &mut output); } output } fn render_node( node: ego_tree::NodeRef<'_, scraper::node::Node>, base_url: &Url, images: &[ArticleImage], output: &mut String, ) { use scraper::node::Node; match node.value() { Node::Text(text) => output.push_str(&escape_html(text)), Node::Element(element) => { let tag = element.name(); if matches!( tag, "script" | "style" | "iframe" | "form" | "nav" | "aside" ) { return; } if tag == "img" { render_image(element, base_url, images, output); return; } if matches!(tag, "br" | "hr") { output.push('<'); output.push_str(tag); output.push_str(" />"); return; } let allowed = matches!( tag, "p" | "div" | "section" | "article" | "header" | "footer" | "h1" | "h2" | "h3" | "h4" | "h5" | "h6" | "ul" | "ol" | "li" | "blockquote" | "pre" | "code" | "strong" | "b" | "em" | "i" | "u" | "s" | "sup" | "sub" | "figure" | "figcaption" | "table" | "thead" | "tbody" | "tfoot" | "tr" | "th" | "td" | "a" | "span" ); if allowed { output.push('<'); output.push_str(tag); if tag == "a" && let Some(href) = element.attr("href") && let Ok(url) = base_url.join(href) && matches!(url.scheme(), "http" | "https") { output.push_str(" href=\""); output.push_str(&escape_html(url.as_str())); output.push('"'); } output.push('>'); } for child in node.children() { render_node(child, base_url, images, output); } if allowed { output.push_str("'); } } _ => { for child in node.children() { render_node(child, base_url, images, output); } } } } fn render_image( element: &scraper::node::Element, base_url: &Url, images: &[ArticleImage], output: &mut String, ) { let Some(source) = element .attr("data-src") .or_else(|| element.attr("data-lazy-src")) .or_else(|| element.attr("src")) else { return; }; let Ok(url) = base_url.join(source) else { return; }; let Some(image) = images.iter().find(|image| image.source_url == url) else { return; }; output.push_str("\"");"); } fn escape_html(value: &str) -> String { value .replace('&', "&") .replace('<', "<") .replace('>', ">") .replace('"', """) .replace('\'', "'") } fn default_output_path(title: &str, format: Format) -> PathBuf { let mut filename: String = title .chars() .map(|character| match character { '/' | '\\' | '\0' => '_', character if character.is_control() => ' ', character => character, }) .collect(); filename = filename.trim().trim_matches('.').trim().to_owned(); if filename.is_empty() { filename.push_str("article"); } filename.truncate(filename.floor_char_boundary(200)); PathBuf::from(format!("{filename}.{}", extension(format))) } fn extension(format: Format) -> &'static str { match format { Format::Epub => "epub", Format::Mobi => "mobi", } } fn temporary_epub_path(output: &Path) -> PathBuf { let mut temporary = output.as_os_str().to_owned(); temporary.push(format!(".{}.epub", std::process::id())); PathBuf::from(temporary) }