#!/usr/bin/env python3 """ Merge the Gutenberg hits (gutenberg_nature_travel.csv), the Deutsches Textarchiv's genre lists, the Wikisource harvest (wikisource_nature_travel.csv) and the EcoCor picks (ecocor_nature_travel.csv) into one JSON file for the overview page. Pipeline: python gutendex_nature_travel.py && python wikisource_nature_travel.py python ecocor_nature_travel.py python build_overview.py # -> overview/data.json python geolocate.py # adds places, writes overview/places.json python keywords.py # adds keywords Works found in more than one source are kept in each (they are different editions or transcriptions) and marked with the other source in "also". DTA genres used (from deutschestextarchiv.de/list/browse): Belletristik > Reiseliteratur -> travel Wissenschaft > Zoologie, Biologie, Geologie -> natural_history Wissenschaft > Geographie -> travel if the title names a journey, else natural_history """ import argparse import csv import difflib import html import json import re import unicodedata import urllib.request from collections import Counter from pathlib import Path from gutendex_nature_travel import TITLE_RX DTA = "https://www.deutschestextarchiv.de" DTA_GENRES = { "Belletristik": {"Reiseliteratur": ["travel"]}, "Wissenschaft": {"Zoologie": ["natural_history"], "Biologie": ["natural_history"], "Geologie": ["natural_history"], "Geographie": None}, } def fetch_dta(genre): # The DTA serves a JavaScript cookie check first; sending the cookie skips it. req = urllib.request.Request(f"{DTA}/list/browse?genre={genre}", headers={"User-Agent": "nature-travel-overview/1.0", "Cookie": "verified=1"}) with urllib.request.urlopen(req, timeout=120) as r: return r.read().decode("utf-8") def parse_dta_entry(text): """'Forster, Georg: Ansichten vom Niederrhein. Bd. 1. Berlin, 1791.' -> author, title, year""" text = " ".join(html.unescape(text).split()) author, _, rest = text.partition(": ") if ": " in text else ("", "", text) years = re.findall(r"\b(1[4-9]\d\d)\b", rest) title = re.sub(r"\.\s*$", "", rest) # drop the trailing "Place, Year" imprint when present title = re.sub(r"\.\s*[^.]*,\s*\[?1[4-9]\d\d\]?\s*$", "", title) return author.strip("[] "), title, int(years[-1]) if years else None def load_dta(): rows = [] for main, subs in DTA_GENRES.items(): parts = re.split(r'

', fetch_dta(main)) for i in range(1, len(parts), 2): sub = parts[i] if sub not in subs: continue for dta_id, text in re.findall(r'book/show/([^"]+)">\s*([^<]+)', parts[i + 1]): author, title, year = parse_dta_entry(text) cats = subs[sub] if cats is None: # Geographie: decide by title travel = any(rx.search(title) for rx in TITLE_RX["travel"]) cats = ["travel"] if travel else ["natural_history"] rows.append({ "src": "DTA", "id": dta_id, "t": title, "a": author, "b": None, "y": year, "k": "article" if re.search(r"\bIn: ", title) else "book", "l": ["de"], "c": cats, "s": None, "f": [], "g": f"{main} › {sub}", "u": f"{DTA}/book/show/{dta_id}", "r": f"{DTA}/book/view/{dta_id}?p=1", # online reader }) return rows def load_gutenberg(path): rows = [] for r in csv.DictReader(open(path, encoding="utf-8")): evidence = r["evidence"].split(" | ") if r["evidence"] else [] kinds = sorted({e.split(":")[0].split("/")[1] for e in evidence}) rows.append({ "src": "PG", "id": r["id"], "t": r["title"], "a": "; ".join(re.sub(r", (\d{0,4}\??)-(\d{0,4}\??)(?=( \[[^\]]+\])?$)", r" (\1–\2)", a) for a in r["authors"].split("; ")) if r["authors"] else "", "b": int(r["earliest_author_birth"]) if r["earliest_author_birth"] else None, "y": None, "k": "book", "l": r["languages"].split(","), "c": r["category"].split("+"), "s": int(r["score"]), "f": [f for f in r["flags"].split(",") if f], "g": " · ".join(kinds), "sub": r["subjects"], "u": r["url"], "r": f"https://www.gutenberg.org/cache/epub/{r['id']}/pg{r['id']}-images.html", # static online reader }) return rows def load_wikisource(path): if not Path(path).exists(): print(f" ! {path} not found; run wikisource_nature_travel.py first") return [] rows = [] for r in csv.DictReader(open(path, encoding="utf-8")): rows.append({ "src": "WS", "id": r["page"], "t": r["title"], "a": r["author"], "b": None, "y": int(r["year"]) if r["year"] else None, "k": r["type"], "l": ["de"], "c": r["category"].split("+"), "s": None, "f": [], "g": r["source"], "sum": r["summary"], "u": r["url"], "r": r["url"], }) return rows def load_ecocor(path): if not Path(path).exists(): print(f" ! {path} not found; run ecocor_nature_travel.py first") return [] rows = [] for r in csv.DictReader(open(path, encoding="utf-8")): rows.append({ "src": "EC", "id": r["name"], "t": r["title"], "a": r["author"], "b": None, "y": int(r["year"]) if r["year"] else None, "k": "book", "l": [r["corpus"]], "c": r["category"].split("+"), "s": None, "f": [], "g": f"{r['animals']} animal and {r['plants']} plant species annotated", "u": r["url"], "r": r["url"], }) return rows def _norm(s): s = unicodedata.normalize("NFD", s.replace("ſ", "s")) s = "".join(c for c in s if not unicodedata.combining(c)).lower() for a, b in (("th", "t"), ("ey", "ei"), ("ae", "a"), ("oe", "o"), ("ue", "u")): s = s.replace(a, b) # old vs. modern spelling: Thier/Tier, Reyse/Reise return re.sub(r"[^a-z0-9 ]", "", s) STOP = set("the of and a an in on to der die das und von vom im in nach ein eine zu um aus".split()) def _surname(author): first = re.sub(r"\(.*?\)|\[.*?\]", "", author.split(";")[0]) words = _norm(first.split(",")[0] if "," in first else first).split() return words[-1] if words else "" def mark_overlaps(rows): """Mark works that appear in more than one source (same language, same author surname and a similar title).""" # English Gutenberg only overlaps with EcoCor; skip its other authors to keep this fast. ec_authors = {_surname(r["a"]) for r in rows if r["src"] == "EC"} keyed = [(r, _surname(r["a"]), _norm(r["t"])[:60]) for r in rows] keyed = [(r, a, t) for r, a, t in keyed if not (r["src"] == "PG" and "de" not in r["l"] and a not in ec_authors)] for i, (r1, a1, t1) in enumerate(keyed): for r2, a2, t2 in keyed[i + 1:]: if r1["src"] == r2["src"] or not a1 or a1 != a2 or not set(r1["l"]) & set(r2["l"]): continue n = min(len(t1), len(t2)) # ignore trailing subtitles/translator notes w1, w2 = set(t1.split()) - STOP, set(t2.split()) - STOP words = len(w1 & w2) / max(1, len(w1 | w2)) if difflib.SequenceMatcher(None, t1[:n], t2[:n]).ratio() > 0.8 or words >= 0.6: r1.setdefault("also", []).append(r2["src"]) r2.setdefault("also", []).append(r1["src"]) for r in rows: if "also" in r: r["also"] = sorted(set(r["also"])) def main(): ap = argparse.ArgumentParser() ap.add_argument("--csv", default="gutenberg_nature_travel.csv") ap.add_argument("--ws-csv", default="wikisource_nature_travel.csv") ap.add_argument("--ec-csv", default="ecocor_nature_travel.csv") ap.add_argument("-o", "--out", default="overview/data.json") args = ap.parse_args() rows = (load_gutenberg(args.csv) + load_dta() + load_wikisource(args.ws_csv) + load_ecocor(args.ec_csv)) mark_overlaps(rows) Path(args.out).parent.mkdir(parents=True, exist_ok=True) with open(args.out, "w", encoding="utf-8") as f: json.dump(rows, f, ensure_ascii=False, separators=(",", ":")) c = Counter((r["src"], lang, cat) for r in rows for lang in r["l"] if lang in ("en", "de") for cat in r["c"]) print(f"Wrote {len(rows)} works to {args.out}") for r in rows: if "also" in r and r["src"] in ("WS", "EC"): print(f" overlap: {r['src']} {r['a']} | {r['t'][:60]} -> also in {', '.join(r['also'])}") for k, n in sorted(c.items()): print(" ", *k, n) if __name__ == "__main__": main()