# Woningsearch.nl robots.txt
# Last updated: 2026-05-11
# 9.7M+ address pages, 13 languages

# All bots: default rules
User-agent: *
Content-Signal: ai-train=no, search=yes, ai-input=yes
Disallow: /api/
Disallow: /admin/
Disallow: /test/
Disallow: /tests/
Disallow: /go/
# Block legacy `?lang=` URLs (long since superseded by /<lang>/ path prefixes).
# Crawling them just produces "Page with redirect" entries in Search Console
# and burns crawl budget; the indexable variant is /<lang>/<path>.
Disallow: /*?lang=
Disallow: /*&lang=
# /koopwoningen/ and /listings/ rely on noindex meta tag instead of Disallow
# (Disallow prevents crawling, so Google would never see the noindex directive)

# Block known scraper/spam bots completely
User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: MJ12bot
Disallow: /

#User-agent: AhrefsBot
#Disallow: /

#User-agent: SemrushBot
#Disallow: /

User-agent: DotBot
Disallow: /


# Sitemap
Sitemap: https://woningsearch.nl/sitemap.xml

# AI Agent Discovery
# llms.txt and llms-full.txt are discoverable at:
#   https://woningsearch.nl/llms.txt
#   https://woningsearch.nl/llms-full.txt
# Advertised via <link rel="ai-info"> in the HTML head.