1use std::fs::{self, File};
  2use std::io::Cursor;
  3use std::path::{Path, PathBuf};
  4use std::process::Command;
  5use std::sync::atomic::{AtomicU64, Ordering};
  6use std::time::Duration;
  7
  8use anyhow::{Context, Result, bail};
  9use epub_builder::{EpubBuilder, EpubContent, ReferenceType, ZipLibrary};
 10use legible::parse;
 11use reqwest::blocking::Client;
 12use reqwest::header::{ACCEPT, CONTENT_TYPE, USER_AGENT};
 13use scraper::{Html, Selector};
 14use url::Url;
 15
 16// browser user-agent value used for the https requests
 17const USER_AGENT_VALUE: &str = concat!("link2epub/", env!("CARGO_PKG_VERSION"));
 18static TEMPORARY_IMAGE_ID: AtomicU64 = AtomicU64::new(0);
 19const STYLESHEET: &str = r#"
 20body { font-family: serif; line-height: 1.5; margin: 5%; }
 21h1 { line-height: 1.15; margin-bottom: 0.3em; }
 22.byline, .source { color: #555; font-size: 0.9em; }
 23.article p { margin: 0 0 1em; }
 24.article blockquote { border-left: 0.2em solid #999; margin-left: 0; padding-left: 1em; }
 25.article pre { white-space: pre-wrap; }
 26figure { margin: 1.5em 0; text-align: center; }
 27img { height: auto; max-width: 100%; }
 28"#;
 29
 30#[derive(Clone, Copy, Debug, Eq, PartialEq)]
 31pub enum Format {
 32    Epub,
 33    Mobi,
 34}
 35
 36pub struct Options {
 37    pub url: String,
 38    pub format: Format,
 39}
 40
 41#[derive(Debug)]
 42struct Article {
 43    title: String,
 44    author: Option<String>,
 45    content_html: String,
 46    source_url: String,
 47    images: Vec<ArticleImage>,
 48    image_urls: Vec<(Url, String)>,
 49}
 50
 51#[derive(Debug)]
 52struct ArticleImage {
 53    source_url: Url,
 54    path: String,
 55    media_type: &'static str,
 56    bytes: Vec<u8>,
 57}
 58
 59/// download an article and write it in the requested ebook format
 60pub fn convert(options: Options) -> Result<PathBuf> {
 61    let url = validate_url(&options.url)?;
 62    eprintln!("downloading article: {url}");
 63    let article = download_article(&url)?;
 64    let output = default_output_path(&article.title, options.format);
 65    eprintln!("article: {}", article.title);
 66    eprintln!("output: {}", output.display());
 67
 68    match options.format {
 69        Format::Epub => write_epub(&article, &output)?,
 70        Format::Mobi => write_mobi(&article, &output)?,
 71    }
 72    Ok(output)
 73}
 74
 75fn validate_url(input: &str) -> Result<Url> {
 76    let url = Url::parse(input).context("invalid article URL")?;
 77    if !matches!(url.scheme(), "http" | "https") {
 78        bail!("url scheme must be http or https");
 79    }
 80    Ok(url)
 81}
 82
 83/// fetches the page and applies a reader-mode extraction algorithm
 84fn download_article(url: &Url) -> Result<Article> {
 85    let client = Client::builder()
 86        .timeout(Duration::from_secs(30))
 87        .redirect(reqwest::redirect::Policy::limited(10))
 88        .build()
 89        .context("could not initialize http client")?;
 90    let response = client
 91        .get(url.clone())
 92        .header(USER_AGENT, USER_AGENT_VALUE)
 93        .header(ACCEPT, "text/html,application/xhtml+xml")
 94        .send()
 95        .with_context(|| format!("could not download {url}"))?
 96        .error_for_status()
 97        .with_context(|| format!("server returned an error for {url}"))?;
 98    let final_url = response.url().clone();
 99    let html = response
100        .text()
101        .context("could not read the downloaded page")?;
102    if final_url != *url {
103        eprintln!("redirected to: {final_url}");
104    }
105    eprintln!("extracting reader-mode content...");
106    let mut article = extract_article(&html, &final_url)?;
107    let image_count = article.image_urls.len();
108    eprintln!(
109        "found {} article image{}.",
110        image_count,
111        if image_count == 1 { "" } else { "s" }
112    );
113    article.images = download_images(&client, &article.image_urls);
114    article.image_urls.clear();
115    if image_count > 0 {
116        eprintln!("embedded {} of {image_count} images.", article.images.len());
117    }
118    Ok(article)
119}
120
121fn extract_article(html: &str, url: &Url) -> Result<Article> {
122    let extracted = parse(html, Some(url.as_str()), None)
123        .context("could not identify the article content on this page")?;
124    let title = extracted.title.trim().to_owned();
125    if title.is_empty() {
126        bail!("the page did not contain an article title");
127    }
128
129    let image_urls = discover_image_urls(&extracted.content, url);
130    let text = html2text::from_read(extracted.content.as_bytes(), 100)
131        .context("could not convert article HTML to readable text")?;
132    if text.trim().is_empty() {
133        bail!("the page did not contain readable article text");
134    }
135
136    Ok(Article {
137        title,
138        author: extracted.byline.filter(|value| !value.trim().is_empty()),
139        content_html: extracted.content,
140        source_url: url.to_string(),
141        images: Vec::new(),
142        image_urls,
143    })
144}
145
146fn discover_image_urls(html: &str, base_url: &Url) -> Vec<(Url, String)> {
147    let document = Html::parse_fragment(html);
148    let selector = Selector::parse("img").expect("valid image selector");
149    document
150        .select(&selector)
151        .filter_map(|image| {
152            let attributes = image.value();
153            let source = attributes
154                .attr("data-src")
155                .or_else(|| attributes.attr("data-lazy-src"))
156                .or_else(|| attributes.attr("src"))?;
157            let url = base_url.join(source).ok()?;
158            matches!(url.scheme(), "http" | "https")
159                .then(|| (url, attributes.attr("alt").unwrap_or_default().to_owned()))
160        })
161        .collect()
162}
163
164/// downloads supported article images (treats individual broken images as optional)
165fn download_images(client: &Client, candidates: &[(Url, String)]) -> Vec<ArticleImage> {
166    let mut images = Vec::new();
167    for (index, (url, _alt)) in candidates.iter().enumerate() {
168        eprintln!(
169            "[{}/{}] downloading image: {url}",
170            index + 1,
171            candidates.len()
172        );
173        let result = (|| -> Result<ArticleImage> {
174            let response = client
175                .get(url.clone())
176                .header(USER_AGENT, USER_AGENT_VALUE)
177                .send()?
178                .error_for_status()?;
179            let reported_type = response
180                .headers()
181                .get(CONTENT_TYPE)
182                .and_then(|value| value.to_str().ok())
183                .map(|value| {
184                    value
185                        .split(';')
186                        .next()
187                        .unwrap_or(value)
188                        .trim()
189                        .to_ascii_lowercase()
190                });
191            let bytes = response.bytes()?.to_vec();
192            if bytes.is_empty() {
193                bail!("empty image");
194            }
195            let source_type = reported_type
196                .as_deref()
197                .and_then(supported_image_type)
198                .or_else(|| image_type_from_url(url).and_then(supported_image_type))
199                .context("unsupported image type")?;
200            let (bytes, media_type) = if source_type == "image/avif" {
201                eprintln!(
202                    "[{}/{}] converting avif to png...",
203                    index + 1,
204                    candidates.len()
205                );
206                (
207                    avif_to_png(&bytes).context("could not convert avif image to png")?,
208                    "image/png",
209                )
210            } else {
211                (bytes, source_type)
212            };
213            let extension = image_extension(media_type);
214            Ok(ArticleImage {
215                source_url: url.clone(),
216                path: format!("images/image-{}.{}", images.len() + 1, extension),
217                media_type,
218                bytes,
219            })
220        })();
221        match result {
222            Ok(image) => {
223                eprintln!("[{}/{}] image ready.", index + 1, candidates.len());
224                images.push(image);
225            }
226            Err(error) => eprintln!("warning: skipping image {url}: {error:#}"),
227        }
228    }
229    images
230}
231
232/// transcodes avif to the broadly supported png format used inside the ebook.
233fn avif_to_png(bytes: &[u8]) -> Result<Vec<u8>> {
234    match avif_to_png_rust(bytes) {
235        Ok(png) => Ok(png),
236        Err(rust_error) => {
237            eprintln!("pure-rust avif decoder could not handle this image; trying avifdec...");
238            avif_to_png_with_avifdec(bytes).with_context(|| {
239                format!("pure-rust decoder failed ({rust_error}); avifdec fallback also failed")
240            })
241        }
242    }
243}
244
245fn avif_to_png_rust(bytes: &[u8]) -> Result<Vec<u8>> {
246    let image = avif_rust::image_from_bytes(bytes).context("could not decode avif pixels")?;
247    lodepng::encode_memory(
248        &image.rgba,
249        image.width,
250        image.height,
251        lodepng::ColorType::RGBA,
252        8,
253    )
254    .context("could not encode png pixels")
255}
256
257/// uses libavif's reference CLI for bitstreams unsupported by the Rust decoder.
258fn avif_to_png_with_avifdec(bytes: &[u8]) -> Result<Vec<u8>> {
259    let id = TEMPORARY_IMAGE_ID.fetch_add(1, Ordering::Relaxed);
260    let stem = format!("link2epub-{}-{id}", std::process::id());
261    let input = std::env::temp_dir().join(format!("{stem}.avif"));
262    let output = std::env::temp_dir().join(format!("{stem}.png"));
263    fs::write(&input, bytes).context("could not create temporary avif")?;
264    let result = Command::new("avifdec").arg(&input).arg(&output).output();
265    let _ = fs::remove_file(&input);
266
267    let command_output =
268        result.context("avifdec is not available on PATH")?;
269    if !command_output.status.success() {
270        let _ = fs::remove_file(&output);
271        let stderr = String::from_utf8_lossy(&command_output.stderr);
272        bail!("avifdec could not transcode the image: {}", stderr.trim());
273    }
274    let png = fs::read(&output).context("avifdec did not create the temporary png")?;
275    let _ = fs::remove_file(&output);
276    if !png.starts_with(b"\x89png\r\n\x1a\n") {
277        bail!("avifdec produced an invalid png");
278    }
279    Ok(png)
280}
281
282fn supported_image_type(value: &str) -> Option<&'static str> {
283    match value.trim().to_ascii_lowercase().as_str() {
284        "image/jpeg" | "image/jpg" => Some("image/jpeg"),
285        "image/png" => Some("image/png"),
286        "image/gif" => Some("image/gif"),
287        "image/webp" => Some("image/webp"),
288        "image/svg+xml" => Some("image/svg+xml"),
289        "image/avif" => Some("image/avif"),
290        _ => None,
291    }
292}
293
294fn image_type_from_url(url: &Url) -> Option<&'static str> {
295    let path = url.path().to_ascii_lowercase();
296    let extension = path.rsplit('.').next()?;
297    match extension {
298        "jpg" | "jpeg" => Some("image/jpeg"),
299        "png" => Some("image/png"),
300        "gif" => Some("image/gif"),
301        "webp" => Some("image/webp"),
302        "svg" => Some("image/svg+xml"),
303        "avif" => Some("image/avif"),
304        _ => None,
305    }
306}
307
308fn image_extension(media_type: &str) -> &'static str {
309    match media_type {
310        "image/jpeg" => "jpg",
311        "image/png" => "png",
312        "image/gif" => "gif",
313        "image/webp" => "webp",
314        "image/svg+xml" => "svg",
315        "image/avif" => "avif",
316        _ => unreachable!("only supported image types reach this point"),
317    }
318}
319
320fn write_epub(article: &Article, path: &Path) -> Result<()> {
321    eprintln!("building epub: {}", path.display());
322    let file =
323        File::create(path).with_context(|| format!("could not create {}", path.display()))?;
324    build_epub(article, file).with_context(|| format!("could not generate {}", path.display()))
325}
326
327fn build_epub(article: &Article, output: impl std::io::Write + std::io::Seek) -> Result<()> {
328    let author = article.author.as_deref().unwrap_or("unknown author");
329    let content = article_xhtml(article);
330    let mut builder = EpubBuilder::new(ZipLibrary::new()?)?;
331    builder
332        .metadata("title", &article.title)?
333        .metadata("author", author)?
334        .metadata("lang", "en")?
335        .stylesheet(STYLESHEET.as_bytes())?;
336    for image in &article.images {
337        builder.add_resource(
338            &image.path,
339            Cursor::new(image.bytes.as_slice()),
340            image.media_type,
341        )?;
342    }
343    builder.add_content(
344        EpubContent::new("article.xhtml", Cursor::new(content.into_bytes()))
345            .title(&article.title)
346            .reftype(ReferenceType::Text),
347    )?;
348    builder.generate(output)?;
349    Ok(())
350}
351
352/// builds an intermediate epub and delegates mobi encoding to Calibre.
353fn write_mobi(article: &Article, output: &Path) -> Result<()> {
354    let temporary = temporary_epub_path(output);
355    eprintln!("preparing intermediate epub for mobi conversion...");
356    write_epub(article, &temporary)?;
357
358    let conversion = if command_available("ebook-convert") {
359        eprintln!("converting epub to mobi with Calibre...");
360        Command::new("ebook-convert")
361            .arg(&temporary)
362            .arg(output)
363            .status()
364            .context("could not run ebook-convert")
365    } else {
366        let _ = fs::remove_file(&temporary);
367        bail!(
368            "mobi output requires Calibre's ebook-convert on PATH; install Calibre or use --format epub"
369        );
370    };
371
372    let status = conversion?;
373    let _ = fs::remove_file(&temporary);
374    if !status.success() {
375        bail!("mobi converter exited with status {status}");
376    }
377    if !output.is_file() {
378        bail!(
379            "mobi converter succeeded but did not create {}",
380            output.display()
381        );
382    }
383    Ok(())
384}
385
386fn command_available(command: &str) -> bool {
387    Command::new(command)
388        .arg("--version")
389        .output()
390        .is_ok_and(|output| output.status.success())
391}
392
393fn article_xhtml(article: &Article) -> String {
394    let byline = article
395        .author
396        .as_deref()
397        .map_or_else(String::new, |author| {
398            format!("<p class=\"byline\">{}</p>", escape_html(author))
399        });
400    let body = render_article_html(article);
401    format!(
402        r#"<?xml version="1.0" encoding="utf-8"?>
403<!DOCTYPE html>
404<html xmlns="http://www.w3.org/1999/xhtml">
405<head><title>{title}</title><link rel="stylesheet" type="text/css" href="stylesheet.css" /></head>
406<body><h1>{title}</h1>{byline}<p class="source">Source: <a href="{url}">{url}</a></p><div class="article">{body}</div></body>
407</html>"#,
408        title = escape_html(&article.title),
409        url = escape_html(&article.source_url),
410    )
411}
412
413/// Converts reader-mode HTML to a conservative XHTML subset and rewrites images locally.
414fn render_article_html(article: &Article) -> String {
415    let document = Html::parse_fragment(&article.content_html);
416    let base_url = Url::parse(&article.source_url).expect("stored source url is valid");
417    let mut output = String::new();
418    for child in document.tree.root().children() {
419        render_node(child, &base_url, &article.images, &mut output);
420    }
421    output
422}
423
424fn render_node(
425    node: ego_tree::NodeRef<'_, scraper::node::Node>,
426    base_url: &Url,
427    images: &[ArticleImage],
428    output: &mut String,
429) {
430    use scraper::node::Node;
431
432    match node.value() {
433        Node::Text(text) => output.push_str(&escape_html(text)),
434        Node::Element(element) => {
435            let tag = element.name();
436            if matches!(
437                tag,
438                "script" | "style" | "iframe" | "form" | "nav" | "aside"
439            ) {
440                return;
441            }
442            if tag == "img" {
443                render_image(element, base_url, images, output);
444                return;
445            }
446            if matches!(tag, "br" | "hr") {
447                output.push('<');
448                output.push_str(tag);
449                output.push_str(" />");
450                return;
451            }
452
453            let allowed = matches!(
454                tag,
455                "p" | "div"
456                    | "section"
457                    | "article"
458                    | "header"
459                    | "footer"
460                    | "h1"
461                    | "h2"
462                    | "h3"
463                    | "h4"
464                    | "h5"
465                    | "h6"
466                    | "ul"
467                    | "ol"
468                    | "li"
469                    | "blockquote"
470                    | "pre"
471                    | "code"
472                    | "strong"
473                    | "b"
474                    | "em"
475                    | "i"
476                    | "u"
477                    | "s"
478                    | "sup"
479                    | "sub"
480                    | "figure"
481                    | "figcaption"
482                    | "table"
483                    | "thead"
484                    | "tbody"
485                    | "tfoot"
486                    | "tr"
487                    | "th"
488                    | "td"
489                    | "a"
490                    | "span"
491            );
492            if allowed {
493                output.push('<');
494                output.push_str(tag);
495                if tag == "a"
496                    && let Some(href) = element.attr("href")
497                    && let Ok(url) = base_url.join(href)
498                    && matches!(url.scheme(), "http" | "https")
499                {
500                    output.push_str(" href=\"");
501                    output.push_str(&escape_html(url.as_str()));
502                    output.push('"');
503                }
504                output.push('>');
505            }
506            for child in node.children() {
507                render_node(child, base_url, images, output);
508            }
509            if allowed {
510                output.push_str("</");
511                output.push_str(tag);
512                output.push('>');
513            }
514        }
515        _ => {
516            for child in node.children() {
517                render_node(child, base_url, images, output);
518            }
519        }
520    }
521}
522
523fn render_image(
524    element: &scraper::node::Element,
525    base_url: &Url,
526    images: &[ArticleImage],
527    output: &mut String,
528) {
529    let Some(source) = element
530        .attr("data-src")
531        .or_else(|| element.attr("data-lazy-src"))
532        .or_else(|| element.attr("src"))
533    else {
534        return;
535    };
536    let Ok(url) = base_url.join(source) else {
537        return;
538    };
539    let Some(image) = images.iter().find(|image| image.source_url == url) else {
540        return;
541    };
542    output.push_str("<img src=\"");
543    output.push_str(&escape_html(&image.path));
544    output.push_str("\" alt=\"");
545    output.push_str(&escape_html(element.attr("alt").unwrap_or_default()));
546    output.push_str("\" />");
547}
548
549fn escape_html(value: &str) -> String {
550    value
551        .replace('&', "&amp;")
552        .replace('<', "&lt;")
553        .replace('>', "&gt;")
554        .replace('"', "&quot;")
555        .replace('\'', "&#39;")
556}
557
558fn default_output_path(title: &str, format: Format) -> PathBuf {
559    let mut filename: String = title
560        .chars()
561        .map(|character| match character {
562            '/' | '\\' | '\0' => '_',
563            character if character.is_control() => ' ',
564            character => character,
565        })
566        .collect();
567    filename = filename.trim().trim_matches('.').trim().to_owned();
568    if filename.is_empty() {
569        filename.push_str("article");
570    }
571    filename.truncate(filename.floor_char_boundary(200));
572    PathBuf::from(format!("{filename}.{}", extension(format)))
573}
574
575fn extension(format: Format) -> &'static str {
576    match format {
577        Format::Epub => "epub",
578        Format::Mobi => "mobi",
579    }
580}
581
582fn temporary_epub_path(output: &Path) -> PathBuf {
583    let mut temporary = output.as_os_str().to_owned();
584    temporary.push(format!(".{}.epub", std::process::id()));
585    PathBuf::from(temporary)
586}