"""Drop a link instead of a file: _incoming/<anything>.url or .txt holding one http(s) address of a PDF.

  * The file may be plain text (just the address) or a Windows internet shortcut ([InternetShortcut] / URL=...).
  * An arXiv page address (arxiv.org/abs/<id>) is rewritten to the PDF address (arxiv.org/pdf/<id>).
  * The download uses curl: follows redirects, connection and total time limits, a size cap (settings fetch.max_mb;
    enforced by curl and again by a hard file-size limit on the process and a check afterwards).
  * The result must really be a PDF: the bytes must start with %PDF- and pdfinfo must read it. Otherwise the run fails
    and says why (bad address, HTTP error, not a PDF, too large, timeout...). Nothing is committed on failure.
  * The file name comes from the server (Content-Disposition) or the last part of the address, with .pdf added if missing.
The download stays in the scratch folder; the pipeline then stores it as the book's source PDF and records the address in
book.json (source_url). Nothing is published anywhere else."""
import json
import os
import re
import resource
import shutil
import subprocess
from urllib.parse import unquote, urlparse

UA = "Mozilla/5.0 (compatible; audiobook-pipeline/1.0; +drop-folder)"


class FetchError(Exception):
    """The message is what the operator reads in the run log."""


def read_url_file(path):
    """The one http(s) address in a .url/.txt file."""
    with open(path, encoding="utf-8-sig", errors="replace") as f:
        lines = [ln.strip() for ln in f.read().splitlines()]
    found = []
    for ln in lines:
        if not ln or ln.startswith(("#", ";", "[")):
            continue
        if re.match(r"^url\s*=", ln, re.I):
            ln = ln.split("=", 1)[1].strip()
        elif "=" in ln.split()[0] and not ln.lower().startswith(("http://", "https://")):
            continue                      # other shortcut keys (IconIndex=0, ...)
        found.append(ln)
    if not found:
        raise FetchError("%s is empty: put one web address (https://...) in it" % os.path.basename(path))
    if len(set(found)) > 1:
        raise FetchError("%s has %d different lines; it must hold exactly one web address (https://...)" % (os.path.basename(path), len(set(found))))
    url = found[0]
    if not re.match(r"^https?://[^\s/]+", url, re.I) or re.search(r"\s", url):
        raise FetchError("%s: %r is not a web address. It must start with http:// or https:// and contain no spaces" % (os.path.basename(path), url[:120]))
    return url


def normalize_url(url):
    """arxiv.org/abs/<id> -> arxiv.org/pdf/<id>. Everything else is used as it is."""
    m = re.match(r"^https?://(?:www\.|export\.)?arxiv\.org/abs/(.+?)/?(?:[?#].*)?$", url, re.I)
    if m:
        return "https://arxiv.org/pdf/" + m.group(1)
    return url


def safe_name(name):
    name = unquote(name).strip().strip(".")
    name = re.sub(r"[^A-Za-z0-9._() \-]+", "_", os.path.basename(name)).strip(" _")
    if not name:
        name = "download"
    if not name.lower().endswith(".pdf"):
        name += ".pdf"
    return name


def name_from_response(headers_text, final_url):
    """File name from the last Content-Disposition header, else the last part of the final address."""
    found = re.findall(r"^content-disposition:.*$", headers_text, re.I | re.M)
    if found:
        m = re.search(r"filename\*\s*=\s*[^']*'[^']*'([^;\r\n]+)", found[-1], re.I) or re.search(r'filename\s*=\s*"?([^";\r\n]+)"?', found[-1], re.I)
        if m:
            return safe_name(m.group(1))
    path = urlparse(final_url).path.rstrip("/")
    return safe_name(path.rsplit("/", 1)[-1] if path else "")


def check_pdf(path):
    with open(path, "rb") as f:
        head = f.read(2048)
    if b"%PDF-" not in head[:1024]:
        kind = "an HTML page" if re.search(rb"<\s*(!doctype|html|head|body)", head[:512], re.I) else "not a PDF file"
        title = re.search(rb"<title[^>]*>(.*?)</title>", head, re.I | re.S)
        raise FetchError("the link did not return a PDF (the download is %s%s). Use the direct link to the PDF file" %
                         (kind, ": %r" % title.group(1).decode("utf-8", "replace").strip()[:80] if title else ""))
    r = subprocess.run(["pdfinfo", path], capture_output=True, text=True)
    pages = re.search(r"^Pages:\s+(\d+)", r.stdout, re.M)
    if r.returncode != 0 or not pages or int(pages.group(1)) < 1:
        raise FetchError("the download starts like a PDF but pdfinfo cannot read it (damaged or cut short): %s" % (r.stderr.strip()[:200] or "no pages"))
    return int(pages.group(1))


CURL_ERRORS = {
    3: "the web address is malformed", 5: "cannot resolve the proxy", 6: "the host name does not exist (check the address)",
    7: "the server refused the connection", 28: "the download timed out", 35: "TLS handshake failed", 47: "too many redirects",
    51: "the server certificate does not match", 52: "the server sent an empty reply", 56: "the connection broke during the download",
    60: "the server certificate is not trusted", 63: "the file is larger than the allowed maximum",
}


def download(cfg, url, out_dir):
    """Download url into out_dir. Returns {'pdf', 'bytes', 'pages', 'final_url'}. Raises FetchError."""
    fc = cfg.get("fetch", {})
    max_bytes = int(float(fc.get("max_mb", 300)) * 1024 * 1024)
    os.makedirs(out_dir, exist_ok=True)
    part = os.path.join(out_dir, "download.partial")
    hdr = os.path.join(out_dir, "headers.txt")
    for p in (part, hdr):
        if os.path.exists(p):
            os.remove(p)
    cmd = ["curl", "-sS", "-L", "--fail", "--max-redirs", "10", "--connect-timeout", str(fc.get("connect_timeout_s", 30)),
           "--max-time", str(fc.get("max_time_s", 900)), "--max-filesize", str(max_bytes), "-A", UA, "-D", hdr, "-o", part,
           "-w", "%{url_effective}", "--proto", "=http,https", "--proto-redir", "=http,https", url]
    limit = lambda: resource.setrlimit(resource.RLIMIT_FSIZE, (max_bytes + (1 << 20), max_bytes + (1 << 20)))
    try:
        r = subprocess.run(cmd, capture_output=True, text=True, preexec_fn=limit)
    except FileNotFoundError:
        raise FetchError("curl is not installed on this host")
    if r.returncode != 0:
        if os.path.exists(part):
            os.remove(part)
        if r.returncode in (-25, 25) or r.returncode == 63:
            raise FetchError("the file is larger than the allowed maximum of %s MB (settings fetch.max_mb)" % fc.get("max_mb", 300))
        detail = r.stderr.strip().splitlines()[-1] if r.stderr.strip() else "curl exit %d" % r.returncode
        m = re.search(r"returned error: (\d+)", detail)
        why = ("the server answered HTTP %s (%s)" % (m.group(1), {"404": "not found", "403": "forbidden", "401": "login required", "410": "gone"}.get(m.group(1), "error"))
               if m else CURL_ERRORS.get(r.returncode, detail))
        raise FetchError("could not download %s: %s" % (url, why))
    size = os.path.getsize(part)
    if size == 0:
        raise FetchError("the download from %s was empty" % url)
    if size > max_bytes:
        os.remove(part)
        raise FetchError("the file is %.0f MB, larger than the allowed maximum of %s MB (settings fetch.max_mb)" % (size / 1048576, fc.get("max_mb", 300)))
    final_url = r.stdout.strip() or url
    headers = ""
    if os.path.exists(hdr):
        with open(hdr, encoding="utf-8", errors="replace") as f:
            headers = f.read()
    try:
        pages = check_pdf(part)
    except FetchError:
        os.remove(part)
        raise
    pdf = os.path.join(out_dir, name_from_response(headers, final_url))
    shutil.move(part, pdf)
    return {"pdf": pdf, "bytes": size, "pages": pages, "final_url": final_url}


def fetch_from_file(cfg, urlfile, out_dir):
    """Everything for one .url/.txt file. Returns a JSON-able dict: pdf, source_url, original_url, name_hints, bytes, pages."""
    original = read_url_file(urlfile)
    url = normalize_url(original)
    print("link file %s -> %s%s" % (os.path.basename(urlfile), url, "  (rewritten from %s)" % original if url != original else ""))
    got = download(cfg, url, out_dir)
    side = os.path.splitext(urlfile)[0] + ".json"
    if os.path.exists(side):
        shutil.copyfile(side, os.path.splitext(got["pdf"])[0] + ".json")
    print("downloaded %s: %.1f MB, %d pages (valid PDF)" % (os.path.basename(got["pdf"]), got["bytes"] / 1048576, got["pages"]))
    return {"pdf": got["pdf"], "source_url": url,
            "original_url": original, "name_hints": [os.path.basename(urlfile)], "bytes": got["bytes"], "pages": got["pages"]}