HA webhook works, also more makelaars

Fetches 51+ listings from D&S with full details:
- Paginates through /aanbod/koopwoningen
- Extracts property postcode from Google Maps iframe URL
- Parses all kenmerken (features) from detail pages
- Includes price, address, rooms, area, build year, energy label

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>

add_scraper_context.md

@@ -219,6 +219,8 @@ If the CMS is unknown, the tool prints structural diagnostics (card selectors, f
 
 ## Important Notes
 
+Don't treat detail pages as optional, we always want all the info!
+
 ### Status Mapping
 Brokers use different status strings. Always map to one of:
 - `"beschikbaar"` — Available for sale
@@ -270,6 +272,7 @@ The database stores this as JSON in the `extra` column.
 - Nominatim (geocoding) has a 1 req/s limiter built into `huizenbot.py`
 - Never spawn parallel requests without the human's approval
 - Always use the `USER_AGENT` header (includes contact info for respectful scraping)
+- Don't keep curling the same endpoint, pipe it to a <name makelaar>.dump and then rg through it to find what you need. Can also pipe it through the bsprettify.py and then rg that. 
 
 ---
 

makelaars.md

@@ -30,9 +30,9 @@
 | [x] | Ooms Makelaars Schiedam | ooms.com | Gerrit Verboonstraat 2 |
 | [x] | De Witte Garantiemakelaars | dewittegarantiemakelaars.nl | Philippusweg 2 |
 | [x] | Makelaardij Wassenaar | makelaardijwassenaar.nl | Gerrit Verboonstraat 12 |
-| [ ] | 3D Makelaars | 3dmakelaars.nl | Gerrit Verboonstraat 17 |
-| [ ] | Dupont Makelaars | dupont.nl | Rotterdamsedijk 437 |
-| [ ] | D&S Makelaardij | densmakelaars.nl | Land van Belofte 50 |
+| [x] | 3D Makelaars | 3dmakelaars.nl | Gerrit Verboonstraat 17 |
+| [x] | Dupont Makelaars | dupont.nl | Rotterdamsedijk 437 |
+| [x] | D&S Makelaardij | densmakelaars.nl | Land van Belofte 50 |
 | [ ] | Moerman & De Jong Makelaars | moerman-dejong.nl | Lange Kerkstraat 80B |
 | [ ] | Hagestein Makelaardij | — | Degerfors 54 |
 | [ ] | Schieland Borsboom NVM Makelaars | schielandborsboom.nl | (Rotterdam, actief in Schiedam) |

src/adapters/ssr.py

@@ -456,6 +456,406 @@ def _infer_stad(postcode: str | None) -> str | None:
     return None
 
 
+# ---------------------------------------------------------------------------
+# D&S Makelaars (Schiedam)
+# ---------------------------------------------------------------------------
+
+_DS_BASE = "https://www.densmakelaars.nl"
+
+_DS_STATUS_MAP = {
+    "onder bod":    "onder_bod",
+    "te koop":      "beschikbaar",
+    "nieuw":        "beschikbaar",
+    "beschikbaar":  "beschikbaar",
+    "verkocht":     "verkocht",
+}
+
+
+def _ds_detail(detail_url: str, html_text: str = None) -> dict:
+    """Fetch D&S detail page and extract all kenmerken from <dt>/<dd> pairs and postcode from maps URL."""
+    try:
+        # If html_text not provided, fetch it
+        if html_text is None:
+            import httpx
+            r = httpx.get(
+                detail_url,
+                headers={"User-Agent": config.USER_AGENT},
+                timeout=15,
+                follow_redirects=True,
+            )
+            html_text = r.text
+
+        soup = BeautifulSoup(html_text, "html.parser")
+
+        # Parse <dt>/<dd> pairs into a label → value map
+        kv: dict[str, str] = {}
+        dts = soup.select("dt")
+        dds = soup.select("dd")
+
+        for dt, dd in zip(dts, dds):
+            label = dt.get_text(strip=True).lower()
+            value = dd.get_text(strip=True)
+            kv[label] = value
+
+        # Extract postcode from Google Maps URL in iframe src
+        # Pattern: q=...POSTCODE...,CITY where POSTCODE is 4 digits + 2 letters
+        postcode = None
+        m = re.search(r'q=.+?,(\d{4})\s+([A-Z]{2}),', html_text)
+        if m:
+            postcode = f"{m.group(1)}{m.group(2)}"
+
+        # Extract specific fields
+        result = {
+            "status": kv.get("status", "beschikbaar").lower(),
+            "woningtype": kv.get("soort woning"),
+            "bouwjaar": kv.get("bouwjaar"),
+            "woonoppervlak": kv.get("woonoppervlakte"),
+            "kamers": kv.get("aantal kamers"),
+            "slaapkamers": kv.get("aantal slaapkamers"),
+            "energielabel": kv.get("energielabel"),
+            "postcode": postcode,
+        }
+        return result
+    except Exception as e:
+        log.warning("dens: detail fetch fout %s: %s", detail_url, e)
+        return {}
+
+
+def fetch_dens() -> list[RawListing]:
+    """Fetch D&S Makelaars listings with full detail pages."""
+    listings = []
+    page = 1
+
+    while True:
+        url = f"{_DS_BASE}/aanbod/koopwoningen?page={page}"
+        soup = fetch_soup(url)
+        cards = soup.select(".col-12.col-md-4.object-wrapper")
+        if not cards:
+            break
+
+        for card in cards:
+            try:
+                # Extract URL
+                a_tag = card.select_one("a.property")
+                if not a_tag or "href" not in a_tag.attrs:
+                    continue
+                detail_url = a_tag["href"]
+                if not detail_url.startswith("http"):
+                    detail_url = _DS_BASE + detail_url
+
+                # Extract listing page data
+                status_label = _text(card, "span.label") or "beschikbaar"
+                status_label = status_label.strip().lower()
+                status = _DS_STATUS_MAP.get(status_label, "beschikbaar")
+
+                adres = _text(card, "h3")
+                stad = _text(card, "h4")
+                prijs_text = _text(card, "div.price")
+                prijs = parse_prijs(prijs_text)
+
+                # Extract area and rooms from footer
+                footer_spans = card.select("div.footer span")
+                woonoppervlak = None
+                kamers = None
+                for span in footer_spans:
+                    text = span.get_text(strip=True)
+                    if "m²" in text:
+                        woonoppervlak = parse_m2(text)
+                    elif "kamers" in text.lower():
+                        m = re.search(r"(\d+)", text)
+                        if m:
+                            kamers = int(m.group(1))
+
+                # Extract hero image
+                img_tag = card.select_one("img")
+                hero = img_tag["src"] if img_tag else None
+
+                # Fetch and parse detail page
+                detail_data = _ds_detail(detail_url)
+
+                # Use postcode from detail data (extracted from Google Maps URL)
+                postcode = detail_data.get("postcode")
+
+                # Determine status from detail page if available
+                if detail_data.get("status"):
+                    status = _DS_STATUS_MAP.get(detail_data["status"], status)
+
+                # Build listing
+                listings.append(RawListing(
+                    url=detail_url,
+                    source_makelaar="dens",
+                    adres=adres,
+                    postcode=postcode,
+                    stad=stad or _infer_stad(postcode),
+                    prijs=prijs,
+                    status=status,
+                    hero_image_url=hero,
+                    woningtype=detail_data.get("woningtype"),
+                    bouwjaar=int(detail_data["bouwjaar"]) if detail_data.get("bouwjaar") else None,
+                    woonoppervlak=parse_m2(detail_data.get("woonoppervlak")) or woonoppervlak,
+                    kamers=int(detail_data["kamers"]) if detail_data.get("kamers") else kamers,
+                    slaapkamers=int(detail_data["slaapkamers"]) if detail_data.get("slaapkamers") else None,
+                    energielabel=detail_data.get("energielabel"),
+                ))
+            except Exception as e:
+                log.warning("dens: parse fout: %s", e)
+
+        if len(cards) < 10:
+            break
+        page += 1
+
+    log.info("dens: %d listings opgehaald", len(listings))
+    return listings
+
+
+# ---------------------------------------------------------------------------
+# 3D Makelaars (Schiedam/Vlaardingen)
+# ---------------------------------------------------------------------------
+
+_3D_BASE = "https://3dmakelaars.nl"
+
+
+def _3dmakelaars_detail(detail_url: str) -> dict:
+    """Fetch 3dmakelaars detail page and extract structured info block."""
+    try:
+        soup = fetch_soup(detail_url)
+
+        # Parse structured info block: span (label) + p (value) pairs
+        kv: dict[str, str] = {}
+        for li in soup.select("div.tl-adiltional-inforamtion ul.tl-adiltional-listed li"):
+            label_el = li.select_one("span")
+            value_el = li.select_one("p")
+            if label_el and value_el:
+                label = label_el.get_text(strip=True).lower()
+                value = value_el.get_text(strip=True)
+                kv[label] = value
+
+        # Extract postcode from first description paragraph
+        postcode = None
+        p_tag = soup.select_one(".omschrijving > p:nth-child(1)")
+        if p_tag:
+            text = p_tag.get_text()
+            postcode = _extract_postcode(text)
+
+        return {
+            "kamers": int(kv["aantal kamers"].split()[0]) if "aantal kamers" in kv else None,
+            "slaapkamers": int(kv["aantal slaapkamers"].split()[0]) if "aantal slaapkamers" in kv else None,
+            "bouwjaar": int(kv["bouwjaar"]) if "bouwjaar" in kv else None,
+            "woningtype": kv.get("bouwvorm"),
+            "woonoppervlak": parse_m2(kv.get("oppervlakte")),
+            "postcode": postcode,
+        }
+    except Exception as e:
+        log.warning("3dmakelaars: detail fetch fout %s: %s", detail_url, e)
+        return {}
+
+
+def fetch_3dmakelaars() -> list[RawListing]:
+    """Fetch 3D Makelaars listings with pagination."""
+    listings = []
+    page = 1
+
+    while True:
+        url = (
+            f"{_3D_BASE}/woningen-te-koop-in-schiedam-en-vlaardingen"
+            f"?kamers=&oppervlakte=&woonplaats=&video=&prijs=3&page={page}"
+        )
+        soup = fetch_soup(url)
+        cards = soup.select("div.tl-properties-item")
+        if not cards:
+            break
+
+        for card in cards:
+            try:
+                # Extract detail URL from onclick attribute
+                onclick = card.get("onclick", "")
+                detail_url = None
+                if "window.location" in onclick:
+                    m = re.search(r"window\.location\s*=\s*['\"]([^'\"]+)['\"]", onclick)
+                    if m:
+                        detail_url = _3D_BASE + m.group(1)
+
+                if not detail_url:
+                    continue
+
+                # Extract listing-level info
+                adres = _text(card, "h3.price")
+                prijs_text = _text(card, "span.address")
+                prijs = parse_prijs(prijs_text)
+
+                # Extract rooms and area from meta list
+                kamers = None
+                woonoppervlak = None
+                for li in card.select("ul.tl-meta-listed > li"):
+                    text = li.get_text(strip=True)
+                    if "kamers" in text.lower():
+                        m = re.search(r"(\d+)", text)
+                        if m:
+                            kamers = int(m.group(1))
+                    elif "m²" in text or "m2" in text:
+                        woonoppervlak = parse_m2(text)
+
+                # Extract image
+                img_tag = card.select_one("img")
+                hero = img_tag["src"] if img_tag else None
+                if hero and not hero.startswith("http"):
+                    hero = _3D_BASE + hero
+
+                # Fetch detail page for full info
+                detail_data = _3dmakelaars_detail(detail_url)
+
+                # Postcode from detail page, fallback to extraction from address
+                postcode = detail_data.get("postcode")
+                if not postcode and adres:
+                    postcode = _extract_postcode(adres)
+
+                listings.append(RawListing(
+                    url=detail_url,
+                    source_makelaar="3dmakelaars",
+                    adres=adres,
+                    postcode=postcode,
+                    stad=_infer_stad(postcode),
+                    prijs=prijs,
+                    woningtype=detail_data.get("woningtype"),
+                    bouwjaar=detail_data.get("bouwjaar"),
+                    woonoppervlak=woonoppervlak or detail_data.get("woonoppervlak"),
+                    kamers=kamers or detail_data.get("kamers"),
+                    slaapkamers=detail_data.get("slaapkamers"),
+                    hero_image_url=hero,
+                ))
+            except Exception as e:
+                log.warning("3dmakelaars: parse fout: %s", e)
+
+        if len(cards) < 7:
+            break
+        page += 1
+
+    log.info("3dmakelaars: %d listings opgehaald", len(listings))
+    return listings
+
+
+# ---------------------------------------------------------------------------
+# Dupont ERA Makelaars (Schiedam/Rotterdam)
+# ---------------------------------------------------------------------------
+
+_DUPONT_BASE = "https://www.dupont.nl"
+
+_DUPONT_STATUS_MAP = {
+    "te koop":                      "beschikbaar",
+    "nieuw":                        "beschikbaar",
+    "onder bod":                    "onder_bod",
+    "verkocht onder voorbehoud":    "onder_bod",
+    "verkocht":                     "verkocht",
+}
+
+
+def _dupont_detail(detail_url: str) -> dict:
+    """Fetch Dupont detail page and extract kenmerken from dt/dd pairs."""
+    try:
+        soup = fetch_soup(detail_url)
+
+        # Parse dt/dd pairs into label → value map
+        kv: dict[str, str] = {}
+        dts = soup.select("dt")
+        dds = soup.select("dd")
+
+        for dt, dd in zip(dts, dds):
+            label = dt.get_text(strip=True).lower()
+            value = dd.get_text(strip=True)
+            kv[label] = value
+
+        # Extract postcode from small tag (format: "NNNN AA CITY")
+        postcode = None
+        small_tag = soup.select_one("section div.container-fluid small")
+        if small_tag:
+            postcode = _extract_postcode(small_tag.get_text())
+
+        return {
+            "postcode": postcode,
+            "woningtype": kv.get("soort woning"),
+            "bouwjaar": kv.get("bouwjaar"),
+            "woonoppervlak": kv.get("woonoppervlakte"),
+            "kamers": kv.get("aantal kamers"),
+            "slaapkamers": kv.get("aantal slaapkamers"),
+            "energielabel": kv.get("energielabel"),
+        }
+    except Exception as e:
+        log.warning("dupont: detail fetch fout %s: %s", detail_url, e)
+        return {}
+
+
+def fetch_dupont() -> list[RawListing]:
+    """Fetch Dupont ERA Makelaars listings with pagination and detail pages."""
+    listings = []
+    page = 1
+
+    while True:
+        url = f"{_DUPONT_BASE}/aanbod/koopwoningen?page={page}"
+        soup = fetch_soup(url)
+        cards = soup.select("article.object")
+        if not cards:
+            break
+
+        for card in cards:
+            try:
+                # Extract URL
+                a_tag = card.select_one("a[href]")
+                if not a_tag or "href" not in a_tag.attrs:
+                    continue
+                detail_url = a_tag["href"]
+                if not detail_url.startswith("http"):
+                    detail_url = _DUPONT_BASE + detail_url
+
+                # Extract listing-level data
+                adres = _text(card, "h3")
+                stad = _text(card, "h4")
+                prijs_text = _text(card, "div.price")
+                prijs = parse_prijs(prijs_text)
+
+                # Extract status from label
+                status_label = _text(card, "div.label") or "beschikbaar"
+                status_label = status_label.strip().lower()
+                status = _DUPONT_STATUS_MAP.get(status_label, "beschikbaar")
+
+                # Extract image
+                img_tag = card.select_one("img.img-responsive")
+                hero = img_tag["src"] if img_tag else None
+                if hero and not hero.startswith("http"):
+                    hero = _DUPONT_BASE + hero
+
+                # Fetch detail page for full data
+                detail_data = _dupont_detail(detail_url)
+
+                # Use postcode from detail if available
+                postcode = detail_data.get("postcode")
+
+                listings.append(RawListing(
+                    url=detail_url,
+                    source_makelaar="dupont",
+                    adres=adres,
+                    postcode=postcode,
+                    stad=stad or _infer_stad(postcode),
+                    prijs=prijs,
+                    status=status,
+                    hero_image_url=hero,
+                    woningtype=detail_data.get("woningtype"),
+                    bouwjaar=int(detail_data["bouwjaar"]) if detail_data.get("bouwjaar") else None,
+                    woonoppervlak=parse_m2(detail_data.get("woonoppervlak")),
+                    kamers=int(detail_data["kamers"]) if detail_data.get("kamers") else None,
+                    slaapkamers=int(detail_data["slaapkamers"]) if detail_data.get("slaapkamers") else None,
+                    energielabel=detail_data.get("energielabel"),
+                ))
+            except Exception as e:
+                log.warning("dupont: parse fout: %s", e)
+
+        if len(cards) < 10:
+            break
+        page += 1
+
+    log.info("dupont: %d listings opgehaald", len(listings))
+    return listings
+
+
 # ---------------------------------------------------------------------------
 # SCRAPERS — exporteer hier alle actieve SSR adapters
 # ---------------------------------------------------------------------------
@@ -465,4 +865,7 @@ SCRAPERS = {
     'woongoed':    fetch_woongoed,
     'dewittegarantiemakelaars': fetch_dewittegarantiemakelaars,
     'wassenaar':   fetch_wassenaar,
+    'dens':        fetch_dens,
+    '3dmakelaars': fetch_3dmakelaars,
+    'dupont':      fetch_dupont,
 }

tests/test_adapters.py

@@ -10,13 +10,13 @@ from adapters import SCRAPERS
 
 logging.basicConfig(
     stream=sys.stdout,
-    level=logging.DEBUG,
+    level=logging.INFO, # debug costs too many tokens
     format="%(asctime)s %(levelname)s %(name)s — %(message)s",
     datefmt="%Y-%m-%dT%H:%M:%S",
 )
 
 # --- change this to test a different adapter ---
-ADAPTER = SCRAPERS['wassenaar']
+ADAPTER = SCRAPERS['dupont']
 
 if __name__ == "__main__":
     print(f"Testing adapter: {ADAPTER.__name__}")

tests/test_ha.py

@@ -0,0 +1,37 @@
+import sys
+sys.path.insert(0, "../src")
+
+import logging
+
+from huizenbot import notify_ha, RawListing
+
+
+logging.basicConfig(
+    stream=sys.stdout,
+    level=logging.INFO, # debug costs too many tokens
+    format="%(asctime)s %(levelname)s %(name)s — %(message)s",
+    datefmt="%Y-%m-%dT%H:%M:%S",
+)
+
+
+TEST_LISTING = RawListing(
+    url="https://home.kalsbeek.dev/api/webhook/new_house",
+    source_makelaar="test",
+    adres="Teststraat 1",
+    stad="Delft",
+    postcode="2613AA",
+    prijs=350000,
+    hero_image_url=None,
+)
+
+TEST_TRAVEL = {
+    "fiets_persoon1": 20,
+    "fiets_persoon2": 35,
+    "ov_persoon1":    30,
+    "ov_persoon2":    45,
+}
+
+if __name__ == "__main__":
+    print("=== Home Assistant webhook ===")
+    notify_ha(TEST_LISTING, TEST_TRAVEL)
+    print("  verstuurd (check HA voor bevestiging)")