#!/usr/bin/env python3
"""
Collect travel writing and natural history from German Wikisource via the MediaWiki API.

Usage:
    python wikisource_nature_travel.py              # -> wikisource_nature_travel.csv

Reads every page in the categories below (one level of subcategories, minus the
Meyers/Pauly encyclopedia entries and journal index pages) and parses the page's
metadata template ({{Textdaten}}, {{GartenlaubenArtikel}}, …) for author, title,
year and source. Each work is typed as a book (standalone publication) or an article (in a periodical
or collection).
Only needs the standard library.
"""
import argparse
import csv
import json
import re
import sys
import time
import urllib.parse
import urllib.request

API = "https://de.wikisource.org/w/api.php"
UA = {"User-Agent": "nature-travel-catalogue/1.0 (https://thomashaider.net/natural-histories/)"}
CATEGORIES = {
    "travel": ["Reisebericht"],
    "natural_history": ["Zoologie", "Entomologie", "Ichthyologie", "Malakologie", "Ornithologie",
                        "Botanik", "Mykologie", "Biologie", "Naturschutz"],
}
SKIP_TITLE = re.compile(r"^(RE:|Meyers|ADB:|BLKÖ:|Zedler:|Pierer:|Brockhaus:|Herders:)")


def api(**params):
    params.update(format="json", formatversion="2")
    url = API + "?" + urllib.parse.urlencode(params)
    for attempt in range(4):
        try:
            with urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=60) as r:
                return json.load(r)
        except Exception as e:
            print(f"  ! {e}; retrying", file=sys.stderr)
            time.sleep(3 * 2 ** attempt)
    raise RuntimeError(url)


def category_pages(cat):
    params = dict(action="query", list="categorymembers", cmtitle=f"Kategorie:{cat}",
                  cmnamespace=0, cmlimit=500)
    while True:
        d = api(**params)
        yield from (m["title"] for m in d["query"]["categorymembers"])
        if "continue" not in d:
            return
        params.update(d["continue"])


def wikitext(titles):
    """Yield (title, wikitext) for up to 50 titles per request."""
    for i in range(0, len(titles), 50):
        d = api(action="query", prop="revisions", rvprop="content", rvslots="main",
                titles="|".join(titles[i:i + 50]))
        for p in d["query"]["pages"]:
            if "revisions" in p:
                yield p["title"], p["revisions"][0]["slots"]["main"]["content"]
        time.sleep(0.5)


def strip_markup(s):
    s = re.sub(r"\[\[(?:[^|\]]*\|)?([^\]]*)\]\]", r"\1", s)   # [[link|text]] -> text
    s = re.sub(r"\{\{[^{}]*\}\}", "", s)                      # simple templates
    s = re.sub(r"<[^>]+>|'''?", "", s)
    return " ".join(s.split()).strip(" .")


def parse_header(text):
    m = re.search(r"\{\{\s*([^|}\n]+?)\s*\n?\|(.*?)\n\}\}", text, re.S)
    if not m:
        return None, {}
    fields = {}
    for line in re.split(r"\n\s*\|", "\n|" + m.group(2)):
        k, _, v = line.partition("=")
        if k.strip():
            fields[k.strip().upper()] = v.strip()
    return m.group(1).strip(), fields


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("-o", "--out", default="wikisource_nature_travel.csv")
    args = ap.parse_args()

    # Chapters of a multi-page work are subpages ("Work/Chapter"); collapse them into the work.
    cats = {}
    for cat_label, names in CATEGORIES.items():
        for name in names:
            for t in category_pages(name):
                if not SKIP_TITLE.match(t):
                    cats.setdefault(t.split("/")[0], set()).add(cat_label)
    print(f"{len(cats)} pages in the categories", file=sys.stderr)

    rows = []
    for title, text in wikitext(sorted(cats)):
        template, f = parse_header(text)
        author = strip_markup(f.get("AUTOR", "")) or "unbekannt"
        year = re.search(r"1[4-9]\d\d", f.get("ERSCHEINUNGSJAHR", "") or f.get("JAHR", "")
                         or f.get("ENTSTEHUNGSJAHR", ""))
        # HERKUNFT names the periodical or collection a piece appeared in; "off" marks a
        # standalone publication.
        source = strip_markup(f.get("HERKUNFT", "")).replace("&nbsp;", " ")
        if source.lower() == "off":
            source = ""
        if template and template.startswith("GartenlaubenArtikel"):
            source = f"Die Gartenlaube ({f.get('JAHR', '').strip()})"
        is_article = bool(source)
        rows.append({
            "title": strip_markup(f.get("TITEL", "")) or title,
            "page": title,
            "author": author,
            "year": int(year.group(0)) if year else "",
            "type": "article" if is_article else "book",
            "source": source,
            "category": "+".join(sorted(cats[title])),
            "summary": strip_markup(f.get("KURZBESCHREIBUNG", "")),
            "url": "https://de.wikisource.org/wiki/" + urllib.parse.quote(title.replace(" ", "_")),
        })

    with open(args.out, "w", newline="", encoding="utf-8") as fh:
        w = csv.DictWriter(fh, fieldnames=list(rows[0]))
        w.writeheader()
        w.writerows(sorted(rows, key=lambda r: (r["year"] or 9999, r["title"])))
    from collections import Counter
    c = Counter((r["type"], r["category"]) for r in rows)
    print(f"Wrote {len(rows)} works to {args.out}", file=sys.stderr)
    for k, n in sorted(c.items()):
        print("  ", *k, n, file=sys.stderr)


if __name__ == "__main__":
    main()