#!/usr/bin/env python3
"""
Scrape NaturaDB wildlife interaction data and enrich HerbAPI species.
"""

import json
import re
import time
import urllib.request
import urllib.error
import sys

HERBAPI_BASE = "http://herbapi01.corp.sub-net.at:8080/api/v1"
HERBAPI_TOKEN = "km2WjhgyMTHlltwgch5TZADHQ-4uIg0NxBeowD-DHGk"
NATURADB_BASE = "https://www.naturadb.de/pflanzen"
USER_AGENT = "Mozilla/5.0 (compatible; HerbAPI-Enrichment/1.0; +https://sub-net.at)"
DELAY = 0.5


def api_get(path):
    """GET from HerbAPI."""
    url = f"{HERBAPI_BASE}{path}"
    req = urllib.request.Request(url)
    req.add_header("Authorization", f"Bearer {HERBAPI_TOKEN}")
    req.add_header("Accept", "application/json")
    with urllib.request.urlopen(req) as resp:
        return json.loads(resp.read().decode())


def api_put(path, data):
    """PUT to HerbAPI."""
    url = f"{HERBAPI_BASE}{path}"
    body = json.dumps(data).encode()
    req = urllib.request.Request(url, data=body, method="PUT")
    req.add_header("Authorization", f"Bearer {HERBAPI_TOKEN}")
    req.add_header("Content-Type", "application/json")
    req.add_header("Accept", "application/json")
    with urllib.request.urlopen(req) as resp:
        return json.loads(resp.read().decode())


def fetch_naturadb(latin_name):
    """Fetch a NaturaDB plant page. Returns HTML string or None."""
    slug = latin_name.lower().replace(" ", "-")
    url = f"{NATURADB_BASE}/{slug}/"
    req = urllib.request.Request(url)
    req.add_header("User-Agent", USER_AGENT)
    try:
        with urllib.request.urlopen(req, timeout=15) as resp:
            return resp.read().decode("utf-8", errors="replace")
    except urllib.error.HTTPError as e:
        if e.code == 404:
            return None
        print(f"  HTTP {e.code} for {url}")
        return None
    except Exception as e:
        print(f"  Error fetching {url}: {e}")
        return None


def extract_td_value(html, label):
    """Extract value from <td>label:</td><td>value</td> pattern."""
    pattern = rf"<td>{re.escape(label)}:?</td>\s*<td[^>]*>(.*?)</td>"
    m = re.search(pattern, html, re.DOTALL)
    if m:
        # Strip HTML tags from value
        val = re.sub(r"<[^>]+>", "", m.group(1)).strip()
        return val
    return None


def extract_native_status(html):
    """Extract native status from chip badges."""
    # Look for the primary native status chips (large, colored)
    statuses = []
    for m in re.finditer(
        r'chip--large\s+chip--no-border\s+clr-white\s+bg-\w+"[^>]*>([^<]+)', html
    ):
        tag = m.group(1).strip()
        if tag in (
            "heimische Wildform",
            "Archäophyt",
            "Neophyt",
            "nicht heimisch (Neophyt)",
        ):
            statuses.append(tag)
    return statuses


def extract_badge_tags(html):
    """Extract ecological badge chips (large, plain text)."""
    tags = []
    for m in re.finditer(r'chip--large\s+clr-text"[^>]*>([^<]+)', html):
        tag = m.group(1).strip()
        if tag and tag not in ("", "winterhart"):
            tags.append(tag)
    return tags


def parse_count(text):
    """Extract leading integer from text like '82 (Nektar und/oder ...)' """
    if not text:
        return None
    m = re.match(r"(\d+)", text.strip())
    return int(m.group(1)) if m else None


def parse_specialist_count(text):
    """Extract specialist/spezialisiert count from text like '39 (davon 5 spezialisiert)'."""
    if not text:
        return None
    m = re.search(r"davon\s+(\d+)\s+spezialisiert", text)
    return int(m.group(1)) if m else None


def parse_nectar_pollen(text):
    """Extract numeric value from '2/4 - mäßig' -> 2."""
    if not text:
        return None
    m = re.match(r"(\d+)/4", text.strip())
    return int(m.group(1)) if m else None


def build_wildlife_value(data):
    """Build a structured wildlife_value string from scraped data."""
    parts = []

    # Nectar and pollen
    np_parts = []
    if data.get("nectar") is not None:
        np_parts.append(f"Nectar: {data['nectar']}/4")
    if data.get("pollen") is not None:
        np_parts.append(f"Pollen: {data['pollen']}/4")
    if np_parts:
        parts.append(", ".join(np_parts) + ".")

    # Wild bees
    if data.get("wildbienen_count") is not None:
        s = f"Supports {data['wildbienen_count']} wild bee species"
        if data.get("wildbienen_specialists") is not None:
            s += f" ({data['wildbienen_specialists']} specialists)"
        parts.append(s + ".")

    # Butterflies / moths
    if data.get("schmetterlinge_count") is not None:
        s = f"{data['schmetterlinge_count']} butterfly/moth species"
        if data.get("raupen_count") is not None:
            spec = ""
            if data.get("raupen_specialists") is not None:
                spec = f" ({data['raupen_specialists']} specialized)"
            s += f", {data['raupen_count']} as caterpillar host{spec}"
        parts.append(s + ".")

    # Hoverflies
    if data.get("schwebfliegen_count") is not None:
        parts.append(f"{data['schwebfliegen_count']} hoverfly species.")

    # Beetles
    if data.get("kaefer_count") is not None:
        parts.append(f"{data['kaefer_count']} beetle species.")

    # Birds
    if data.get("vogelarten_count") is not None:
        parts.append(f"{data['vogelarten_count']} bird species.")

    # Mammals
    if data.get("saeugetier_count") is not None:
        parts.append(f"{data['saeugetier_count']} mammal species.")

    # Native status
    if data.get("native_status"):
        parts.append(" ".join(data["native_status"]) + ".")

    # Notable badges
    notable = [
        t
        for t in data.get("badges", [])
        if any(
            kw in t.lower()
            for kw in [
                "insektenpflanze",
                "raupenfutter",
                "vogelschutz",
                "vogelnähr",
                "bienenweide",
            ]
        )
    ]
    if notable:
        parts.append("Tags: " + ", ".join(notable) + ".")

    return " ".join(parts) if parts else None


def scrape_species(html):
    """Parse NaturaDB HTML and return structured wildlife data dict."""
    data = {}

    # Nectar and pollen values
    nectar_raw = extract_td_value(html, "Nektarwert")
    pollen_raw = extract_td_value(html, "Pollenwert")
    data["nectar"] = parse_nectar_pollen(nectar_raw)
    data["pollen"] = parse_nectar_pollen(pollen_raw)

    # Wild bees
    bees_raw = extract_td_value(html, "Wildbienen")
    data["wildbienen_count"] = parse_count(bees_raw)
    data["wildbienen_specialists"] = parse_specialist_count(bees_raw)

    # Butterflies/moths
    schmett_raw = extract_td_value(html, "Schmetterlinge")
    data["schmetterlinge_count"] = parse_count(schmett_raw)

    # Caterpillar hosts
    raupen_raw = extract_td_value(html, "Raupen")
    data["raupen_count"] = parse_count(raupen_raw)
    data["raupen_specialists"] = parse_specialist_count(raupen_raw)

    # Hoverflies
    schweb_raw = extract_td_value(html, "Schwebfliegen")
    data["schwebfliegen_count"] = parse_count(schweb_raw)

    # Beetles
    kaefer_raw = extract_td_value(html, "Käfer")
    data["kaefer_count"] = parse_count(kaefer_raw)

    # Birds
    vogel_raw = extract_td_value(html, "fressende Vogelarten")
    data["vogelarten_count"] = parse_count(vogel_raw)

    # Mammals
    saeuget_raw = extract_td_value(html, "fressende Säugetierarten")
    data["saeugetier_count"] = parse_count(saeuget_raw)

    # Native status
    data["native_status"] = extract_native_status(html)

    # Badge tags
    data["badges"] = extract_badge_tags(html)

    return data


def has_any_data(data):
    """Check if we scraped anything meaningful."""
    for k, v in data.items():
        if k in ("native_status", "badges"):
            if v:
                return True
        elif v is not None:
            return True
    return False


def main():
    print("Fetching species list from HerbAPI...")
    species_list = api_get("/species?per_page=200")["data"]
    print(f"Found {len(species_list)} species.\n")

    enriched = 0
    skipped_has_data = 0
    skipped_not_found = 0
    skipped_no_data = 0
    errors = 0

    for i, sp in enumerate(species_list):
        slug = sp["slug"]
        name = sp["name_scientific"]
        existing_wv = sp.get("wildlife_value")

        # Only enrich if wildlife_value is empty/null
        if existing_wv:
            print(f"[{i+1:3d}/{len(species_list)}] {slug:40s} SKIP (already has data)")
            skipped_has_data += 1
            continue

        print(f"[{i+1:3d}/{len(species_list)}] {slug:40s} ", end="", flush=True)

        # Fetch NaturaDB page
        html = fetch_naturadb(name)
        time.sleep(DELAY)

        if html is None:
            print("NOT FOUND on NaturaDB")
            skipped_not_found += 1
            continue

        # Parse wildlife data
        data = scrape_species(html)

        if not has_any_data(data):
            print("no wildlife data on page")
            skipped_no_data += 1
            continue

        # Build wildlife_value string
        wildlife_value = build_wildlife_value(data)
        if not wildlife_value:
            print("no wildlife data extracted")
            skipped_no_data += 1
            continue

        # GET full species, merge, PUT back
        try:
            full = api_get(f"/species/{slug}")
            full["wildlife_value"] = wildlife_value

            # Remove read-only / computed fields that the PUT endpoint might reject
            for key in ("created_at", "updated_at", "family"):
                full.pop(key, None)

            api_put(f"/species/{full['id']}", full)
            print(f"ENRICHED -> {wildlife_value[:80]}...")
            enriched += 1
        except Exception as e:
            print(f"API ERROR: {e}")
            errors += 1

    print("\n" + "=" * 70)
    print(f"DONE. Results:")
    print(f"  Enriched:           {enriched}")
    print(f"  Already had data:   {skipped_has_data}")
    print(f"  Not on NaturaDB:    {skipped_not_found}")
    print(f"  No wildlife data:   {skipped_no_data}")
    print(f"  Errors:             {errors}")
    print(f"  Total:              {len(species_list)}")


if __name__ == "__main__":
    main()