331 lines
10 KiB
Python
331 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Scrape NaturaDB wildlife interaction data and enrich HerbAPI species.
|
|
"""
|
|
|
|
import json
|
|
import re
|
|
import time
|
|
import urllib.request
|
|
import urllib.error
|
|
import sys
|
|
|
|
HERBAPI_BASE = "http://herbapi01.corp.sub-net.at:8080/api/v1"
|
|
HERBAPI_TOKEN = "km2WjhgyMTHlltwgch5TZADHQ-4uIg0NxBeowD-DHGk"
|
|
NATURADB_BASE = "https://www.naturadb.de/pflanzen"
|
|
USER_AGENT = "Mozilla/5.0 (compatible; HerbAPI-Enrichment/1.0; +https://sub-net.at)"
|
|
DELAY = 0.5
|
|
|
|
|
|
def api_get(path):
|
|
"""GET from HerbAPI."""
|
|
url = f"{HERBAPI_BASE}{path}"
|
|
req = urllib.request.Request(url)
|
|
req.add_header("Authorization", f"Bearer {HERBAPI_TOKEN}")
|
|
req.add_header("Accept", "application/json")
|
|
with urllib.request.urlopen(req) as resp:
|
|
return json.loads(resp.read().decode())
|
|
|
|
|
|
def api_put(path, data):
|
|
"""PUT to HerbAPI."""
|
|
url = f"{HERBAPI_BASE}{path}"
|
|
body = json.dumps(data).encode()
|
|
req = urllib.request.Request(url, data=body, method="PUT")
|
|
req.add_header("Authorization", f"Bearer {HERBAPI_TOKEN}")
|
|
req.add_header("Content-Type", "application/json")
|
|
req.add_header("Accept", "application/json")
|
|
with urllib.request.urlopen(req) as resp:
|
|
return json.loads(resp.read().decode())
|
|
|
|
|
|
def fetch_naturadb(latin_name):
|
|
"""Fetch a NaturaDB plant page. Returns HTML string or None."""
|
|
slug = latin_name.lower().replace(" ", "-")
|
|
url = f"{NATURADB_BASE}/{slug}/"
|
|
req = urllib.request.Request(url)
|
|
req.add_header("User-Agent", USER_AGENT)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=15) as resp:
|
|
return resp.read().decode("utf-8", errors="replace")
|
|
except urllib.error.HTTPError as e:
|
|
if e.code == 404:
|
|
return None
|
|
print(f" HTTP {e.code} for {url}")
|
|
return None
|
|
except Exception as e:
|
|
print(f" Error fetching {url}: {e}")
|
|
return None
|
|
|
|
|
|
def extract_td_value(html, label):
|
|
"""Extract value from <td>label:</td><td>value</td> pattern."""
|
|
pattern = rf"<td>{re.escape(label)}:?</td>\s*<td[^>]*>(.*?)</td>"
|
|
m = re.search(pattern, html, re.DOTALL)
|
|
if m:
|
|
# Strip HTML tags from value
|
|
val = re.sub(r"<[^>]+>", "", m.group(1)).strip()
|
|
return val
|
|
return None
|
|
|
|
|
|
def extract_native_status(html):
|
|
"""Extract native status from chip badges."""
|
|
# Look for the primary native status chips (large, colored)
|
|
statuses = []
|
|
for m in re.finditer(
|
|
r'chip--large\s+chip--no-border\s+clr-white\s+bg-\w+"[^>]*>([^<]+)', html
|
|
):
|
|
tag = m.group(1).strip()
|
|
if tag in (
|
|
"heimische Wildform",
|
|
"Archäophyt",
|
|
"Neophyt",
|
|
"nicht heimisch (Neophyt)",
|
|
):
|
|
statuses.append(tag)
|
|
return statuses
|
|
|
|
|
|
def extract_badge_tags(html):
|
|
"""Extract ecological badge chips (large, plain text)."""
|
|
tags = []
|
|
for m in re.finditer(r'chip--large\s+clr-text"[^>]*>([^<]+)', html):
|
|
tag = m.group(1).strip()
|
|
if tag and tag not in ("", "winterhart"):
|
|
tags.append(tag)
|
|
return tags
|
|
|
|
|
|
def parse_count(text):
|
|
"""Extract leading integer from text like '82 (Nektar und/oder ...)' """
|
|
if not text:
|
|
return None
|
|
m = re.match(r"(\d+)", text.strip())
|
|
return int(m.group(1)) if m else None
|
|
|
|
|
|
def parse_specialist_count(text):
|
|
"""Extract specialist/spezialisiert count from text like '39 (davon 5 spezialisiert)'."""
|
|
if not text:
|
|
return None
|
|
m = re.search(r"davon\s+(\d+)\s+spezialisiert", text)
|
|
return int(m.group(1)) if m else None
|
|
|
|
|
|
def parse_nectar_pollen(text):
|
|
"""Extract numeric value from '2/4 - mäßig' -> 2."""
|
|
if not text:
|
|
return None
|
|
m = re.match(r"(\d+)/4", text.strip())
|
|
return int(m.group(1)) if m else None
|
|
|
|
|
|
def build_wildlife_value(data):
|
|
"""Build a structured wildlife_value string from scraped data."""
|
|
parts = []
|
|
|
|
# Nectar and pollen
|
|
np_parts = []
|
|
if data.get("nectar") is not None:
|
|
np_parts.append(f"Nectar: {data['nectar']}/4")
|
|
if data.get("pollen") is not None:
|
|
np_parts.append(f"Pollen: {data['pollen']}/4")
|
|
if np_parts:
|
|
parts.append(", ".join(np_parts) + ".")
|
|
|
|
# Wild bees
|
|
if data.get("wildbienen_count") is not None:
|
|
s = f"Supports {data['wildbienen_count']} wild bee species"
|
|
if data.get("wildbienen_specialists") is not None:
|
|
s += f" ({data['wildbienen_specialists']} specialists)"
|
|
parts.append(s + ".")
|
|
|
|
# Butterflies / moths
|
|
if data.get("schmetterlinge_count") is not None:
|
|
s = f"{data['schmetterlinge_count']} butterfly/moth species"
|
|
if data.get("raupen_count") is not None:
|
|
spec = ""
|
|
if data.get("raupen_specialists") is not None:
|
|
spec = f" ({data['raupen_specialists']} specialized)"
|
|
s += f", {data['raupen_count']} as caterpillar host{spec}"
|
|
parts.append(s + ".")
|
|
|
|
# Hoverflies
|
|
if data.get("schwebfliegen_count") is not None:
|
|
parts.append(f"{data['schwebfliegen_count']} hoverfly species.")
|
|
|
|
# Beetles
|
|
if data.get("kaefer_count") is not None:
|
|
parts.append(f"{data['kaefer_count']} beetle species.")
|
|
|
|
# Birds
|
|
if data.get("vogelarten_count") is not None:
|
|
parts.append(f"{data['vogelarten_count']} bird species.")
|
|
|
|
# Mammals
|
|
if data.get("saeugetier_count") is not None:
|
|
parts.append(f"{data['saeugetier_count']} mammal species.")
|
|
|
|
# Native status
|
|
if data.get("native_status"):
|
|
parts.append(" ".join(data["native_status"]) + ".")
|
|
|
|
# Notable badges
|
|
notable = [
|
|
t
|
|
for t in data.get("badges", [])
|
|
if any(
|
|
kw in t.lower()
|
|
for kw in [
|
|
"insektenpflanze",
|
|
"raupenfutter",
|
|
"vogelschutz",
|
|
"vogelnähr",
|
|
"bienenweide",
|
|
]
|
|
)
|
|
]
|
|
if notable:
|
|
parts.append("Tags: " + ", ".join(notable) + ".")
|
|
|
|
return " ".join(parts) if parts else None
|
|
|
|
|
|
def scrape_species(html):
|
|
"""Parse NaturaDB HTML and return structured wildlife data dict."""
|
|
data = {}
|
|
|
|
# Nectar and pollen values
|
|
nectar_raw = extract_td_value(html, "Nektarwert")
|
|
pollen_raw = extract_td_value(html, "Pollenwert")
|
|
data["nectar"] = parse_nectar_pollen(nectar_raw)
|
|
data["pollen"] = parse_nectar_pollen(pollen_raw)
|
|
|
|
# Wild bees
|
|
bees_raw = extract_td_value(html, "Wildbienen")
|
|
data["wildbienen_count"] = parse_count(bees_raw)
|
|
data["wildbienen_specialists"] = parse_specialist_count(bees_raw)
|
|
|
|
# Butterflies/moths
|
|
schmett_raw = extract_td_value(html, "Schmetterlinge")
|
|
data["schmetterlinge_count"] = parse_count(schmett_raw)
|
|
|
|
# Caterpillar hosts
|
|
raupen_raw = extract_td_value(html, "Raupen")
|
|
data["raupen_count"] = parse_count(raupen_raw)
|
|
data["raupen_specialists"] = parse_specialist_count(raupen_raw)
|
|
|
|
# Hoverflies
|
|
schweb_raw = extract_td_value(html, "Schwebfliegen")
|
|
data["schwebfliegen_count"] = parse_count(schweb_raw)
|
|
|
|
# Beetles
|
|
kaefer_raw = extract_td_value(html, "Käfer")
|
|
data["kaefer_count"] = parse_count(kaefer_raw)
|
|
|
|
# Birds
|
|
vogel_raw = extract_td_value(html, "fressende Vogelarten")
|
|
data["vogelarten_count"] = parse_count(vogel_raw)
|
|
|
|
# Mammals
|
|
saeuget_raw = extract_td_value(html, "fressende Säugetierarten")
|
|
data["saeugetier_count"] = parse_count(saeuget_raw)
|
|
|
|
# Native status
|
|
data["native_status"] = extract_native_status(html)
|
|
|
|
# Badge tags
|
|
data["badges"] = extract_badge_tags(html)
|
|
|
|
return data
|
|
|
|
|
|
def has_any_data(data):
|
|
"""Check if we scraped anything meaningful."""
|
|
for k, v in data.items():
|
|
if k in ("native_status", "badges"):
|
|
if v:
|
|
return True
|
|
elif v is not None:
|
|
return True
|
|
return False
|
|
|
|
|
|
def main():
|
|
print("Fetching species list from HerbAPI...")
|
|
species_list = api_get("/species?per_page=200")["data"]
|
|
print(f"Found {len(species_list)} species.\n")
|
|
|
|
enriched = 0
|
|
skipped_has_data = 0
|
|
skipped_not_found = 0
|
|
skipped_no_data = 0
|
|
errors = 0
|
|
|
|
for i, sp in enumerate(species_list):
|
|
slug = sp["slug"]
|
|
name = sp["name_scientific"]
|
|
existing_wv = sp.get("wildlife_value")
|
|
|
|
# Only enrich if wildlife_value is empty/null
|
|
if existing_wv:
|
|
print(f"[{i+1:3d}/{len(species_list)}] {slug:40s} SKIP (already has data)")
|
|
skipped_has_data += 1
|
|
continue
|
|
|
|
print(f"[{i+1:3d}/{len(species_list)}] {slug:40s} ", end="", flush=True)
|
|
|
|
# Fetch NaturaDB page
|
|
html = fetch_naturadb(name)
|
|
time.sleep(DELAY)
|
|
|
|
if html is None:
|
|
print("NOT FOUND on NaturaDB")
|
|
skipped_not_found += 1
|
|
continue
|
|
|
|
# Parse wildlife data
|
|
data = scrape_species(html)
|
|
|
|
if not has_any_data(data):
|
|
print("no wildlife data on page")
|
|
skipped_no_data += 1
|
|
continue
|
|
|
|
# Build wildlife_value string
|
|
wildlife_value = build_wildlife_value(data)
|
|
if not wildlife_value:
|
|
print("no wildlife data extracted")
|
|
skipped_no_data += 1
|
|
continue
|
|
|
|
# GET full species, merge, PUT back
|
|
try:
|
|
full = api_get(f"/species/{slug}")
|
|
full["wildlife_value"] = wildlife_value
|
|
|
|
# Remove read-only / computed fields that the PUT endpoint might reject
|
|
for key in ("created_at", "updated_at", "family"):
|
|
full.pop(key, None)
|
|
|
|
api_put(f"/species/{full['id']}", full)
|
|
print(f"ENRICHED -> {wildlife_value[:80]}...")
|
|
enriched += 1
|
|
except Exception as e:
|
|
print(f"API ERROR: {e}")
|
|
errors += 1
|
|
|
|
print("\n" + "=" * 70)
|
|
print(f"DONE. Results:")
|
|
print(f" Enriched: {enriched}")
|
|
print(f" Already had data: {skipped_has_data}")
|
|
print(f" Not on NaturaDB: {skipped_not_found}")
|
|
print(f" No wildlife data: {skipped_no_data}")
|
|
print(f" Errors: {errors}")
|
|
print(f" Total: {len(species_list)}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|