# Block SemrushBot entirely
User-agent: SemrushBot
Disallow: /

# --- Aggressive AI / scraper swarm (#1706) ---
# These crawlers hit the faceted wine search as a distributed swarm and
# saturated the DB/Solr connection path (MySQL #1045 auth-handshake aborts +
# Solr "not available" bursts during business hours). Slow them down and keep
# them off the expensive search-result endpoints. Googlebot and bingbot are
# deliberately NOT in this group — they keep full crawl access for SEO. This is
# belt-and-suspenders for the bots that honour robots.txt; the nginx aggregate
# `crawlers` limit_req zone enforces the same cap for the ones that ignore it.
User-agent: Bytespider
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: CCBot
User-agent: Amazonbot
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: DotBot
User-agent: DataForSeoBot
User-agent: ImagesiftBot
User-agent: Diffbot
User-agent: GoogleOther
User-agent: Timpibot
User-agent: Omgilibot
User-agent: YouBot
Crawl-delay: 30
Disallow: /typo3/
Disallow: /*/weinsuche/
Disallow: /*/solr-suche/
Disallow: /*/recherche-de-vins/
Disallow: /*/degustationsnotizen/

# bingbot — SEO-relevant, kept crawlable; only lightly throttled
User-agent: bingbot
Crawl-delay: 5
Disallow: /typo3/

# Default rules (humans, Googlebot, and any bot not listed above)
User-agent: *
Crawl-delay: 2
Disallow: /typo3/

Sitemap: https://www.vinum.eu/sitemap.xml