"""A reference Caute processor: finds a paper's DOI and suggests its details (ADR 0128).

Standard library only, Python 3.10 or later. One process does everything:

    python processor.py --caute https://api.getcaute.com --public https://doi.example.com
    python processor.py --caute http://api.caute.localhost:8000            # a development server

It serves its own OAuth client metadata document at <public>/client.json, so Caute knows its name and
where to send the person back, and listens for webhooks at <public>/webhook. Run it, open the sign-in
address it prints, choose the spaces it may read, and it sets up its webhook. From then on, each card that
becomes searchable is read, and if its text names a DOI, the record is fetched from doi.org (Crossref or
DataCite, by content negotiation) and Caute is sent suggestions:

    doi, title, authors, year and venue properties, and a note with the citation.

Every one is a suggestion: nothing changes a filter or an export until the person keeps it, and a
dismissed one is never suggested again (INV-50). The processor keeps its credentials and the webhook's
secret in a file only its user can read (--state). It is an example, not a service Axionary Labs runs.
"""

import argparse
import base64
import hashlib
import html
import importlib.util
import json
import os
import re
import secrets
import sys
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path

NAME = "DOI finder"
SCOPES = "items:read items:write notes:write"
# How much of a card's text to look through: a paper names its DOI on its first page.
SCAN_CHARS = 20_000
DOI = re.compile(r"\b(10\.\d{4,9}/[^\s\"'<>]+)", re.I)


def _verifier():
    """docs/webhooks/verify.py, beside this folder in Caute's repository; copy it next to this file
    elsewhere."""
    here = Path(__file__).resolve().parent
    for path in (here / "verify.py", here.parents[2] / "webhooks" / "verify.py"):
        if path.exists():
            spec = importlib.util.spec_from_file_location("caute_verify", path)
            module = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(module)
            return module
    sys.exit("verify.py is missing: copy docs/webhooks/verify.py next to this file.")


verify = _verifier()


# What a card says, and what doi.org says about it.

def find_doi(text: str) -> str | None:
    """The first DOI in the text, without the punctuation a sentence puts after it."""
    m = DOI.search(text[:SCAN_CHARS])
    if not m:
        return None
    return m.group(1).rstrip(".,;:)]}").lower()


ARXIV = re.compile(r"^/(?:abs|pdf|html)/(\d{4}\.\d{4,5})(?:v\d+)?(?:\.pdf)?/?$")


def doi_from_url(url: str | None) -> str | None:
    """A DOI in the card's address (a doi.org link, a publisher's address that names one), or the DOI arXiv
    gives every paper, which its pages show beside the text rather than in it."""
    if not url:
        return None
    parts = urllib.parse.urlsplit(url)
    host = (parts.hostname or "").removeprefix("www.")
    if host == "arxiv.org" and (m := ARXIV.match(parts.path)):
        return f"10.48550/arxiv.{m.group(1)}"
    return find_doi(urllib.parse.unquote(parts.path + " " + parts.query.replace("&", " ").replace("=", " ")))


def _plain(value: str) -> str:
    """Record text is someone else's: no markup, one line (INV-19)."""
    return " ".join(html.unescape(re.sub(r"<[^>]+>", "", value)).split())


def details(record: dict) -> dict:
    """Properties from a CSL-JSON record: title, authors, year, venue. Missing fields are left out."""
    out: dict = {}
    title = record.get("title")
    if isinstance(title, list):
        title = title[0] if title else None
    if isinstance(title, str) and _plain(title):
        out["title"] = _plain(title)[:500]
    authors = []
    for a in record.get("author") or []:
        name = " ".join(p for p in (a.get("given"), a.get("family")) if p) or a.get("literal") or ""
        if _plain(name):
            authors.append(_plain(name)[:200])
    if authors:
        out["authors"] = authors[:50]
    for field in ("issued", "published-print", "published-online", "created"):
        parts = (record.get(field) or {}).get("date-parts") or [[None]]
        if parts and parts[0] and isinstance(parts[0][0], int):
            out["year"] = parts[0][0]
            break
    venue = record.get("container-title") or record.get("publisher")
    if isinstance(venue, list):
        venue = venue[0] if venue else None
    if isinstance(venue, str) and _plain(venue):
        out["venue"] = _plain(venue)[:500]
    return out


def citation(doi: str, found: dict) -> str:
    """A citation note, in Markdown: authors (year). Title. Venue. The DOI's address."""
    authors = found.get("authors") or []
    who = ", ".join(authors[:3]) + (" and others" if len(authors) > 3 else "")
    head = " ".join(p for p in (who, f"({found['year']})." if "year" in found else ("." if who else "")) if p)
    rest = " ".join(f"{found[k]}." for k in ("title", "venue") if k in found)
    return f"{head} {rest} https://doi.org/{doi}".strip()


def look_up(doi: str, mailto: str | None = None) -> dict | None:
    """The record for a DOI from doi.org, as CSL-JSON, or None when it has none."""
    agent = f"caute-doi-processor/1 ({'mailto:' + mailto if mailto else 'https://getcaute.com'})"
    req = urllib.request.Request(f"https://doi.org/{urllib.parse.quote(doi, safe='/')}", headers={
        "Accept": "application/vnd.citationstyles.csl+json", "User-Agent": agent})
    try:
        with urllib.request.urlopen(req, timeout=20) as r:
            return json.loads(r.read(2_000_000))
    except urllib.error.HTTPError as e:
        if e.code in (404, 406):
            return None
        raise
    except ValueError:
        return None


def process(item_id: str, caute, lookup=look_up) -> dict | None:
    """Read one card and send Caute its suggestions. Returns what was suggested, or None."""
    card = caute.request("GET", f"/v1/items/{item_id}")
    # A note is someone's writing, not a paper, and the citation this writes is a note: reading notes would
    # have it cite its own citations, each one a new card it then hears about.
    if card.get("kind") == "note":
        return None
    doi = doi_from_url(card.get("url"))
    if not doi:
        reps = caute.request("GET", f"/v1/items/{item_id}/text").get("representations") or []
        doi = find_doi("\n".join(r.get("text") or "" for r in reps))
    if not doi:
        return None
    record = lookup(doi)
    if not record:
        return None
    found = {"doi": doi, **details(record)}
    for key, value in found.items():
        caute.request("PUT", f"/v1/items/{item_id}/properties/{key}",
                      {"value": value, "type": "list" if isinstance(value, list) else None})
    caute.request("POST", "/v1/notes", {"body": citation(doi, found), "annotates": {"item_id": item_id}})
    return found


# Talking to Caute.

class Caute:
    """Calls to Caute's API with the processor's grant, refreshing its access token as it lapses."""

    def __init__(self, state: "State"):
        self.state = state
        self.lock = threading.Lock()

    def _form(self, url: str, form: dict) -> dict:
        req = urllib.request.Request(url, data=urllib.parse.urlencode(form).encode(),
                                     headers={"Content-Type": "application/x-www-form-urlencoded"})
        with urllib.request.urlopen(req, timeout=20) as r:
            return json.loads(r.read())

    def refresh(self) -> None:
        with self.lock:
            s = self.state.data
            if s.get("expires_at", 0) > time.time() + 30:
                return
            got = self._form(s["token_endpoint"], {"grant_type": "refresh_token", "client_id": s["client_id"],
                                                   "refresh_token": s["refresh_token"], "resource": s["resource"]})
            self.state.signed_in(got)

    def request(self, method: str, path: str, body: dict | None = None) -> dict:
        self.refresh()
        data = None if body is None else json.dumps({k: v for k, v in body.items() if v is not None}).encode()
        req = urllib.request.Request(self.state.data["api"] + path, data=data, method=method, headers={
            "Authorization": f"Bearer {self.state.data['access_token']}", "Content-Type": "application/json",
            "Idempotency-Key": secrets.token_hex(16)})
        try:
            with urllib.request.urlopen(req, timeout=30) as r:
                raw = r.read()
        except urllib.error.HTTPError as e:
            if e.code == 401:
                print("Caute refused the processor's access: it was disconnected. Sign in again.", flush=True)
                self.state.forget()
            raise
        return json.loads(raw) if raw else {}


class State:
    """Credentials and the webhook's secret, in a file only this user can read."""

    def __init__(self, path: Path):
        self.path = path
        self.data = json.loads(path.read_text()) if path.exists() else {}

    def save(self) -> None:
        self.path.parent.mkdir(parents=True, exist_ok=True)
        fd = os.open(self.path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
        with os.fdopen(fd, "w") as f:
            json.dump(self.data, f, indent=2)

    def signed_in(self, got: dict) -> None:
        self.data.update(access_token=got["access_token"], refresh_token=got["refresh_token"],
                         expires_at=time.time() + int(got.get("expires_in", 300)))
        self.save()

    def forget(self) -> None:
        for k in ("access_token", "refresh_token", "expires_at", "webhook_id", "webhook_secret"):
            self.data.pop(k, None)
        self.save()


def discover(api: str) -> dict:
    def get(url):
        with urllib.request.urlopen(url, timeout=20) as r:
            return json.loads(r.read())
    resource = get(f"{api}/.well-known/oauth-protected-resource")
    issuer = resource["authorization_servers"][0].rstrip("/")
    meta = get(f"{issuer}/.well-known/oauth-authorization-server")
    return {"resource": resource["resource"], "authorize": meta["authorization_endpoint"],
            "token_endpoint": meta["token_endpoint"]}


# The web server: the client metadata document, the sign-in hand-back, and the webhook.

def serve(args) -> None:
    public = args.public.rstrip("/")
    state = State(Path(args.state).expanduser())
    client_id = f"{public}/client.json"
    redirect = f"{public}/callback"
    where = discover(args.caute.rstrip("/"))
    state.data.update(api=args.caute.rstrip("/"), client_id=client_id, **{
        k: where[k] for k in ("resource", "token_endpoint")})
    state.save()
    caute = Caute(state)
    pending: dict = {}
    seen: set = set()

    def sign_in_address() -> str:
        verifier = secrets.token_urlsafe(48)
        challenge = base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()).decode().rstrip("=")
        st = secrets.token_urlsafe(16)
        pending[st] = verifier
        return where["authorize"] + "?" + urllib.parse.urlencode({
            "response_type": "code", "client_id": client_id, "redirect_uri": redirect, "scope": SCOPES,
            "code_challenge": challenge, "code_challenge_method": "S256", "state": st,
            "resource": where["resource"]})

    def subscribe() -> None:
        made = caute.request("POST", "/v1/webhooks", {"url": f"{public}/webhook", "types": ["item.searchable"],
                                                      "description": NAME})
        state.data.update(webhook_id=made["id"], webhook_secret=made["secret"])
        state.save()
        print(f"Listening for new cards (webhook {made['id']}).", flush=True)

    def handle(item_id: str) -> None:
        try:
            found = process(item_id, caute, lambda doi: look_up(doi, args.mailto))
            print(f"{item_id}: " + (f"suggested {', '.join(found)}" if found else "nothing to suggest"), flush=True)
        except urllib.error.HTTPError as e:
            # Deleted since, or moved out of the spaces it may read: nothing to do.
            print(f"{item_id}: " + ("gone" if e.code == 404 else str(e)), flush=True)
        except Exception as e:  # noqa: BLE001 - one card's failure shouldn't stop the next
            print(f"{item_id}: {e}", flush=True)

    class Handler(BaseHTTPRequestHandler):
        def _send(self, status: int, body: bytes = b"", kind: str = "text/plain; charset=utf-8") -> None:
            self.send_response(status)
            self.send_header("Content-Type", kind)
            self.send_header("Content-Length", str(len(body)))
            self.end_headers()
            self.wfile.write(body)

        def log_message(self, *a) -> None:
            pass

        def do_GET(self) -> None:
            url = urllib.parse.urlsplit(self.path)
            if url.path == "/client.json":
                doc = {"client_id": client_id, "client_name": NAME, "client_uri": public,
                       "redirect_uris": [redirect], "grant_types": ["authorization_code", "refresh_token"],
                       "response_types": ["code"], "token_endpoint_auth_method": "none", "scope": SCOPES}
                return self._send(200, json.dumps(doc).encode(), "application/json")
            if url.path == "/":
                if state.data.get("refresh_token"):
                    return self._send(200, b"Signed in to Caute and listening for new cards.")
                self.send_response(302)
                self.send_header("Location", sign_in_address())
                self.end_headers()
                return
            if url.path == "/callback":
                q = dict(urllib.parse.parse_qsl(url.query))
                verifier = pending.pop(q.get("state", ""), None)
                if not verifier or "code" not in q:
                    return self._send(400, b"That sign-in didn't come from here. Start again at /.")
                got = caute._form(where["token_endpoint"], {
                    "grant_type": "authorization_code", "code": q["code"], "client_id": client_id,
                    "code_verifier": verifier, "redirect_uri": redirect, "resource": where["resource"]})
                state.signed_in(got)
                subscribe()
                return self._send(200, b"Signed in. You can close this tab.")
            self._send(404)

        def do_POST(self) -> None:
            if self.path != "/webhook" or not state.data.get("webhook_secret"):
                return self._send(404)
            body = self.rfile.read(int(self.headers.get("Content-Length") or 0))
            try:
                event = verify.verify(state.data["webhook_secret"], dict(self.headers.items()), body)
            except verify.Forged:
                return self._send(401)
            # Answer at once, and do the slow part afterwards; a repeat delivery has the same id.
            self._send(204)
            if event["id"] in seen or event["type"] != "item.searchable":
                return
            if (event.get("payload") or {}).get("kind") == "note":
                return  # a note, someone's writing or our own: never read (section 5 of the guide)
            seen.add(event["id"])
            threading.Thread(target=handle, args=(event["subject"]["id"],), daemon=True).start()

    host, port = urllib.parse.urlsplit(public).hostname, args.port or urllib.parse.urlsplit(public).port or 8765
    server = ThreadingHTTPServer((args.listen or host, port), Handler)
    if state.data.get("refresh_token") and not state.data.get("webhook_id"):
        subscribe()
    if not state.data.get("refresh_token"):
        print(f"Sign in to Caute: open {public}/ in a browser.", flush=True)
    else:
        print("Signed in. Listening for new cards.", flush=True)
    server.serve_forever()


def main() -> None:
    p = argparse.ArgumentParser(description="Suggest a paper's details from its DOI (a Caute processor).")
    p.add_argument("--caute", default="https://api.getcaute.com", help="Caute's API address")
    p.add_argument("--public", default="http://127.0.0.1:8765",
                   help="where Caute reaches this processor; https anywhere but your own computer")
    p.add_argument("--listen", help="the address to listen on, if not the public one's host")
    p.add_argument("--port", type=int, help="the port to listen on, if not the public one's")
    p.add_argument("--state", default="~/.caute-doi-processor.json", help="where credentials are kept")
    p.add_argument("--mailto", help="an email address doi.org's providers may contact you at")
    serve(p.parse_args())


if __name__ == "__main__":
    main()