# MS Bautenschutz Wetzlar - Robots.txt for AI & Search Engine Dominance
# Last Updated: 2026-01-27

User-agent: *
Allow: /
Allow: /ratgeber-*
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /tmp/
Disallow: /internal/
Disallow: /preview/
Disallow: /search/
Allow: /wp-admin/admin-ajax.php

# --- EXPLICIT AI & LLM PERMISSIONS (FOR CITATIONS & SEARCH) ---

# OpenAI (ChatGPT, SearchGPT, GPT-4o)
User-agent: GPTBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: OAI-SearchBot
Allow: /

# Google AI (Gemini, SGE, AI Overviews)
User-agent: Google-Extended
Allow: /
User-agent: GoogleOther
Allow: /
User-agent: Google-LLM
Allow: /
User-agent: Googlebot
Allow: /

# Anthropic (Claude, Claude-Web)
User-agent: Claude-Web
Allow: /
User-agent: ClaudeBot
Allow: /

# Microsoft (Bing Chat, Copilot)
User-agent: Bingbot
Allow: /
User-agent: BingPreview
Allow: /
User-agent: MSNBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Apple (Apple Intelligence, Siri)
User-agent: Applebot
Allow: /
User-agent: Applebot-Extended
Allow: /

# Meta (Meta AI, Llama)
User-agent: Meta-ExternalAgent
Allow: /
User-agent: FacebookBot
Allow: /

# Amazon (Amazon Q, Alexa)
User-agent: Amazonbot
Allow: /

# Common Crawl (Foundational Training Data for almost all LLMs)
User-agent: CCBot
Allow: /

# Other Emerging AI Search & Training Bots
User-agent: YouBot
Allow: /
User-agent: DuckAssist
Allow: /
User-agent: NeevaAI
Allow: /
User-agent: Cohere-ai
Allow: /
User-agent: NVIDIA-Custom-Bot
Allow: /
User-agent: Bytespider
Allow: /
User-agent: Diffbot
Allow: /
User-agent: Omgili
Allow: /
User-agent: iaskspider
Allow: /

# --- SITEMAP REFERENCE ---
Sitemap: https://bautenschutz-wetzlar.de/sitemap.xml