#!/usr/bin/env python3
"""
Find natural history and travel writing in Project Gutenberg (English + German)
and write a scored, deduplicated CSV.

Usage:
    python gutendex_nature_travel.py                       # PG catalog -> gutenberg_nature_travel.csv
    python gutendex_nature_travel.py --source gutendex     # crawl all of Gutendex instead (slow, resumable)
    python gutendex_nature_travel.py --enrich              # add Gutendex summaries/download counts to hits
    python gutendex_nature_travel.py --min-score 1         # also keep weak (title-only) hits
    python gutendex_nature_travel.py --base http://localhost:8000 --source gutendex  # self-hosted Gutendex

Only needs the standard library.

How it works
1. Load *every* en/de book, either from PG's own catalog CSV (one ~6 MB download,
   includes LoC classification) or by crawling every Gutendex page (~2,000 requests,
   cached per page so an interrupted crawl resumes; adds summaries and download counts).
2. Classify locally. Each book collects evidence from four sources:
     subj   LoC subject headings      (weight 2; fiction/poetry/drama headings ignored)
     locc   LoC class QK/QL (2), QH/G (1); catalog source only
     shelf  PG bookshelves            (weight 2 for specific shelves, 1 for broad ones)
     title  EN/DE title keywords      (weight 1)
   A book is kept per category when its score reaches --min-score (default 2), so a title
   keyword alone is not enough but "Reise" + Travel Writing shelf is.
3. Fiction (subject "... -- Fiction", LoC class PZ) is flagged and dropped unless --keep-fiction.

Every row lists its evidence, so precision can be checked and the rules below tuned.
"""
import argparse
import csv
import gzip
import io
import json
import re
import sys
import time
import urllib.parse
import urllib.request
from collections import Counter
from pathlib import Path

CATALOG_URL = "https://www.gutenberg.org/cache/epub/feeds/pg_catalog.csv.gz"
UA = {"User-Agent": "gutendex-nature-travel/2.0"}

# --- Classification rules ------------------------------------------------------
# Subject patterns match at the start of a heading or of a "--" subdivision,
# e.g. "Birds -- Great Britain" or "Africa, Central -- Description and travel".
SUBJECT_RULES = {
    "natural_history": [
        r"natural history", r"naturalists?", r"nature study", r"nature\s*(--|$)",
        r"zoology", r"botany", r"plants\b", r"birds\b", r"ornithology", r"bird watching",
        r"insects\b", r"entomology", r"butterflies", r"moths\b", r"beetles\b", r"bees\b",
        r"ants\b", r"spiders\b", r"mammals\b", r"reptiles\b", r"amphibians\b", r"fishes\b",
        r"shells\b", r"mollusks\b", r"marine (biology|animals)", r"animals\b(?!, mythical)",
        r"animal behavior", r"wildlife", r"trees\b", r"wild flowers", r"flowers\b",
        r"ferns\b", r"mosses\b", r"fungi\b", r"mushrooms\b", r"evolution \(biology\)",
        r"natural selection", r"ecology",
    ],
    "travel": [
        r"voyages and travels", r"description and travel", r"discovery and exploration",
        r"voyages around the world", r"travel\b", r"travel(l)?ers?\b", r"explorers\b",
        r"scientific expeditions", r"overland journeys", r"arctic regions", r"antarctica",
        r"mountaineering",
    ],
}
# Subjects that mention plants/animals but are about lore or symbolism, not nature.
SUBJECT_IGNORE = re.compile(r"--\s*(folklore|mythology|symbolic aspects|religious aspects)|mythical", re.I)
# LoC classes (catalog only): (prefix, weight). QK botany and QL zoology are specific;
# QH (natural history *and* general biology) and G (geography, atlases, voyages) are broad.
LOCC_RULES = {"natural_history": [("QK", 2), ("QL", 2), ("QH", 1)], "travel": [("G", 1)]}
# Bookshelves: (pattern, weight). "Category: ..." shelves are PG's broad genre shelves.
SHELF_RULES = {
    "natural_history": [
        (r"^natural history$", 2), (r"^botany$", 2), (r"^zoology$", 2),
        (r"^animals-wild-trapping", 1), (r"^animals-wild(?!-trapping)", 2), (r"^birds, illustrated", 2), (r"^the botanical magazine$", 2),
        (r"entomology", 2), (r"^bird-lore$", 2),
        (r"^category: nature/gardening/animals$", 1), (r"^category: science - biology$", 1),
        (r"^animal$", 1), (r"^biology$", 1),
    ],
    "travel": [
        (r"^category: travel writing$", 1), (r"^travel$", 2), (r"travel journals", 2),
    ],
}
TITLE_RULES = {
    "natural_history": [
        r"natural history", r"naturalist", r"\bbirds?\b", r"\binsects?\b", r"\bbotan",
        r"\bzoolog", r"\bwild flowers\b", r"\bbutterfl",
        r"naturgeschichte", r"naturforscher", r"t(h)?ierleben", r"t(h)?iere\b", r"\bv(ö|oe)gel",
        r"pflanzen", r"insekten", r"k(ä|ae)fer", r"schmetterling",
    ],
    "travel": [
        r"\bvoyages?\b", r"\btravels?\b", r"\bjourney", r"\brambles?\b", r"\bwanderings?\b",
        r"\bexpedition", r"\bexploration", r"\btour\b", r"\bletters from\b",
        r"(?<!\bg)(?<!\bk)(?<!\bp)reise", r"fahrt(en)?\b", r"wanderung", r"streifz(ü|ue)ge", r"\bbriefe aus\b",  # also compounds: Fußreise, Seefahrt
        r"\bforschungs", r"\bin (afrika|amerika|asien|australien|indien|china|japan)\b",
    ],
}
FICTION_SUBJECT = re.compile(r"(--|^)\s*(juvenile )?(fiction|poetry|drama)\s*$", re.I)
# Informational flags, not used for scoring.
FLAG_RULES = {"juvenile": r"juvenile literature", "periodical": r"periodicals",
              "guidebook": r"guidebooks", "early_work": r"early works to 1800"}

SUBJECT_RX = {c: [re.compile(r"(^|--\s*)" + p, re.I) for p in ps] for c, ps in SUBJECT_RULES.items()}
SHELF_RX = {c: [(re.compile(p, re.I), w) for p, w in ps] for c, ps in SHELF_RULES.items()}
TITLE_RX = {c: [re.compile(p, re.I) for p in ps] for c, ps in TITLE_RULES.items()}


# --- HTTP ---------------------------------------------------------------------
def fetch(url, retries=4, timeout=60, raw=False):
    for attempt in range(retries):
        try:
            req = urllib.request.Request(url, headers=UA)
            with urllib.request.urlopen(req, timeout=timeout) as r:
                return r.read() if raw else json.load(r)
        except Exception as e:  # network hiccups, 5xx, timeouts
            wait = 2 ** attempt * 3
            print(f"  ! {e} — retrying in {wait}s", file=sys.stderr)
            time.sleep(wait)
    raise RuntimeError(f"Failed after {retries} attempts: {url}")


# --- Sources: both yield normalized records -----------------------------------
def load_catalog(cache_dir, languages, refresh):
    path = cache_dir / "pg_catalog.csv.gz"
    if refresh or not path.exists():
        print(f"Downloading {CATALOG_URL} …", file=sys.stderr)
        path.write_bytes(fetch(CATALOG_URL, raw=True, timeout=300))
    with gzip.open(path, "rt", encoding="utf-8", newline="") as f:
        for r in csv.DictReader(f):
            langs = [l for l in r["Language"].split("; ") if l]
            if r["Type"] != "Text" or not set(langs) & languages:
                continue
            yield {
                "id": int(r["Text#"]),
                "title": " ".join(r["Title"].split()),
                "authors": [a for a in r["Authors"].split("; ") if a],
                "languages": langs,
                "subjects": [s for s in r["Subjects"].split("; ") if s],
                "locc": [c for c in r["LoCC"].split("; ") if c],
                "bookshelves": [s for s in r["Bookshelves"].split("; ") if s],
                "issued": r["Issued"],
                "summary": "",
                "download_count": "",
            }


def normalize_gutendex(b):
    authors = []
    for a in b.get("authors", []):
        by, dy = a.get("birth_year"), a.get("death_year")
        span = f", {by if by is not None else ''}-{dy if dy is not None else ''}" if (by or dy) else ""
        authors.append(a.get("name", "") + span)
    return {
        "id": b["id"],
        "title": " ".join(b.get("title", "").split()),
        "authors": authors,
        "languages": b.get("languages", []),
        "subjects": b.get("subjects", []),
        "locc": [],  # Gutendex does not expose LoC classes
        "bookshelves": b.get("bookshelves", []),
        "issued": "",
        "summary": " ".join(b.get("summaries", [])),
        "download_count": b.get("download_count", ""),
    }


def crawl_gutendex(base, cache_dir, languages, delay):
    """Crawl every page per language, caching pages to disk so the crawl can resume.
    sort=ascending keeps page boundaries stable between runs."""
    for lang in sorted(languages):
        d = cache_dir / "gutendex" / lang
        d.mkdir(parents=True, exist_ok=True)
        page = 1
        while True:
            p = d / f"page_{page:05d}.json"
            if p.exists():
                data = json.loads(p.read_text(encoding="utf-8"))
            else:
                url = f"{base}/books/?" + urllib.parse.urlencode(
                    {"languages": lang, "sort": "ascending", "page": page})
                data = fetch(url, retries=6, timeout=120)
                p.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
                pages = -(-data.get("count", 0) // 32)
                print(f"  {lang} page {page}/{pages}", file=sys.stderr)
                time.sleep(delay)
            for b in data.get("results", []):
                yield normalize_gutendex(b)
            if not data.get("next"):
                break
            page += 1


def enrich(hits, base, delay):
    """Add Gutendex summaries and download counts to catalog hits (32 ids per request)."""
    ids = [h["id"] for h in hits if not h["summary"]]
    by_id = {h["id"]: h for h in hits}
    failures = 0
    for i in range(0, len(ids), 32):
        chunk = ids[i:i + 32]
        try:
            data = fetch(f"{base}/books/?ids=" + ",".join(map(str, chunk)), retries=3, timeout=120)
            failures = 0
        except RuntimeError as e:
            print(f"  ! enrichment skipped for ids {chunk[0]}…{chunk[-1]}: {e}", file=sys.stderr)
            failures += 1
            if failures == 3:
                print("  ! Gutendex seems down; giving up on enrichment", file=sys.stderr)
                return
            continue
        for b in data.get("results", []):
            if b["id"] in by_id:
                by_id[b["id"]]["summary"] = " ".join(b.get("summaries", []))
                by_id[b["id"]]["download_count"] = b.get("download_count", "")
        print(f"  enriched {min(i + 32, len(ids))}/{len(ids)}", file=sys.stderr)
        time.sleep(delay)


# --- Classification -------------------------------------------------------------
def classify(book):
    """Return ({category: score}, [evidence strings], is_fiction, [flags])."""
    scores, evidence = Counter(), []
    subjects = [s for s in book["subjects"] if not FICTION_SUBJECT.search(s)]
    fiction = len(subjects) < len(book["subjects"]) or any(c.startswith("PZ") for c in book["locc"])
    flags = [f for f, p in FLAG_RULES.items() if any(re.search(p, s, re.I) for s in book["subjects"])]
    subjects = [s for s in subjects if not SUBJECT_IGNORE.search(s)]
    for cat in SUBJECT_RULES:
        hits = [s for s in subjects if any(rx.search(s) for rx in SUBJECT_RX[cat])]
        if hits:
            scores[cat] += 2
            evidence += [f"{cat}/subj:{s}" for s in hits]
        # match the whole class letters: "G" must not match GA/GN/GV…
        locc_hits = [(c, w) for c in book["locc"] for pre, w in LOCC_RULES[cat]
                     if re.match(pre + r"(\d|$)", c)]
        if locc_hits:
            scores[cat] += max(w for _, w in locc_hits)
            evidence += [f"{cat}/locc:{c}" for c, _ in locc_hits]
        shelf_hits = [(s, w) for s in book["bookshelves"] for rx, w in SHELF_RX[cat] if rx.search(s)]
        if shelf_hits:
            scores[cat] += max(w for _, w in shelf_hits)
            evidence += [f"{cat}/shelf:{s}" for s, _ in shelf_hits]
        title_hits = [m.group(0) for rx in TITLE_RX[cat] if (m := rx.search(book["title"]))]
        if title_hits:
            scores[cat] += 1
            evidence += [f"{cat}/title:{t}" for t in title_hits]
    return scores, evidence, fiction, flags


# --- Main -------------------------------------------------------------------------
def main():
    ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
    ap.add_argument("-o", "--out", default="gutenberg_nature_travel.csv")
    ap.add_argument("--languages", default="en,de")
    ap.add_argument("--source", choices=["catalog", "gutendex"], default="catalog")
    ap.add_argument("--base", default="https://gutendex.com")
    ap.add_argument("--cache", default="cache", help="directory for catalog/crawled pages")
    ap.add_argument("--refresh", action="store_true", help="re-download the PG catalog")
    ap.add_argument("--enrich", action="store_true",
                    help="fetch Gutendex summaries + download counts for hits (catalog source)")
    ap.add_argument("--delay", type=float, default=0.5, help="seconds between Gutendex requests")
    ap.add_argument("--min-score", type=int, default=2)
    ap.add_argument("--keep-fiction", action="store_true")
    args = ap.parse_args()

    base = args.base.rstrip("/")
    languages = set(args.languages.split(","))
    cache_dir = Path(args.cache)
    cache_dir.mkdir(parents=True, exist_ok=True)

    books = (load_catalog(cache_dir, languages, args.refresh) if args.source == "catalog"
             else crawl_gutendex(base, cache_dir, languages, args.delay))

    hits, seen, n_total, n_fiction = [], set(), 0, 0
    for b in books:
        if b["id"] in seen:  # Gutendex: bilingual books appear under both languages
            continue
        seen.add(b["id"])
        n_total += 1
        scores, evidence, fiction, flags = classify(b)
        cats = sorted(c for c, s in scores.items() if s >= args.min_score)
        if not cats:
            continue
        if fiction and not args.keep_fiction:
            n_fiction += 1
            continue
        b.update(categories=cats, score=max(scores[c] for c in cats), evidence=evidence,
                 fiction=fiction, flags=flags)
        hits.append(b)
    print(f"Scanned {n_total} books; {len(hits)} hits ({n_fiction} fiction dropped)", file=sys.stderr)

    if args.enrich and args.source == "catalog":
        enrich(hits, base, args.delay)

    fields = ["id", "title", "authors", "earliest_author_birth", "languages", "category", "score",
              "evidence", "fiction", "flags", "subjects", "locc", "bookshelves", "download_count",
              "summary", "url", "txt_url"]
    with open(args.out, "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=fields)
        w.writeheader()
        for b in sorted(hits, key=lambda b: b["id"]):
            births = [int(m.group(1)) for a in b["authors"] if (m := re.search(r", (\d{3,4})\??-", a))]
            w.writerow({
                "id": b["id"],
                "title": b["title"],
                "authors": "; ".join(b["authors"]),
                "earliest_author_birth": min(births) if births else "",
                "languages": ",".join(b["languages"]),
                "category": "+".join(b["categories"]),
                "score": b["score"],
                "evidence": " | ".join(b["evidence"]),
                "fiction": int(b["fiction"]),
                "flags": ",".join(b["flags"]),
                "subjects": " | ".join(b["subjects"]),
                "locc": " | ".join(b["locc"]),
                "bookshelves": " | ".join(b["bookshelves"]),
                "download_count": b["download_count"],
                "summary": b["summary"],
                "url": f"https://www.gutenberg.org/ebooks/{b['id']}",
                "txt_url": f"https://www.gutenberg.org/ebooks/{b['id']}.txt.utf-8",
            })

    c = Counter((lang, cat) for b in hits for lang in b["languages"] if lang in languages
                for cat in b["categories"])
    print(f"Wrote {len(hits)} books to {args.out}", file=sys.stderr)
    for (lang, cat), n in sorted(c.items()):
        print(f"  {lang}  {cat:16s} {n}", file=sys.stderr)


if __name__ == "__main__":
    main()