# robots.txt for administrator.de
# Last Updated: 20250814

# ================================================
# LICENSING NOTICE - IMPORTANT
# ================================================
# The use of content from administrator.de for
# AI training or commercial AI applications is
# STRICTLY PROHIBITED without written license agreement.
#
# License inquiries: licence@administrator.de
# Terms of Service: https://administrator.de/help/80 (EN: https://administrator.de/help/80?lang=en)
# Privacy Policy: https://administrator.de/help/79 (EN: https://administrator.de/help/79?lang=en)
# AI Licensing Info: https://administrator.de/help/82 (EN: https://administrator.de/help/82?lang=en)
# ================================================

# ================================================
# AI CRAWLERS - COMPLETELY BLOCKED without license
# ================================================

# OpenAI Crawlers
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: ChatGPT-Bot
User-agent: OpenAIBot
Disallow: /

# Anthropic (Claude) Crawlers
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Claude-Bot
Disallow: /

# Google AI Crawlers
User-agent: Google-Extended
User-agent: Bard-Bot
User-agent: Gemini-Bot
Disallow: /

# Microsoft/Bing AI Crawlers
User-agent: BingPreview
User-agent: msnbot-media
User-agent: CopilotBot
Disallow: /

# Meta/Facebook AI Crawlers
User-agent: FacebookBot
User-agent: facebookexternalhit
User-agent: Meta-ExternalAgent
Disallow: /

# Other Major AI Companies
User-agent: CCBot
User-agent: PerplexityBot
User-agent: Amazonbot
User-agent: Applebot-Extended
User-agent: Applebot
User-agent: cohere-ai
User-agent: YouBot
User-agent: Diffbot
Disallow: /

# Social Media AI Crawlers
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: WhatsApp
User-agent: TelegramBot
Disallow: /

# Chinese AI Crawlers
User-agent: Bytespider
User-agent: Baiduspider
User-agent: 360Spider
User-agent: Sogou
User-agent: YisouSpider
User-agent: Bytedance
User-agent: PetalBot
Disallow: /

# SEO/Data Mining Crawlers
# User-agent: SemrushBot
# User-agent: AhrefsBot
User-agent: DataForSeoBot
User-agent: MJ12bot
User-agent: Scrapy
User-agent: magpie-crawler
User-agent: Omgilibot
Disallow: /

# Archive/Research Crawlers (often used for AI training)
User-agent: CommonCrawl
User-agent: WebCrawler
User-agent: ImagesiftBot
Disallow: /

# Generic AI Pattern Blocking
User-agent: *bot*ai*
User-agent: *ai*bot*
User-agent: *crawl*ai*
User-agent: *gpt*
User-agent: *llm*
User-agent: *claude*
User-agent: *gemini*
User-agent: *bard*
Disallow: /

# ================================================
# LEGITIMATE SEARCH ENGINES - ALLOWED
# ================================================
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: YandexBot
User-agent: Slurp
Disallow: /settings/
Disallow: /api/
Disallow: /members/dashboard/
Disallow: /members/notifications/
Disallow: /moderators/reports/
Disallow: /connections/
Disallow: /r/
Disallow: /forum/?search=
Disallow: /comments/?search=
Disallow: /members/?search=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?time=
Disallow: /*?lang=
Disallow: /*?action=
Disallow: /*?distance=
Disallow: /*?uri=
Disallow: /*?topic=
Disallow: /*?target=
Disallow: /*?target_id=
Allow: /


# ================================================
# DEFAULT RULES
# ================================================
User-agent: *
Disallow: /settings/
Disallow: /api/
Disallow: /members/dashboard/
Disallow: /members/notifications/
Disallow: /moderators/reports/
Disallow: /connections/
Disallow: /r/
Disallow: /forum/?search=
Disallow: /comments/?search=
Disallow: /members/?search=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?time=
Disallow: /*?lang=
Disallow: /*?action=
Disallow: /*?distance=
Disallow: /*?uri=
Disallow: /*?topic=
Disallow: /*?target=
Disallow: /*?target_id=
Allow: /


# Sitemap
Sitemap: https://administrator.de/sitemap/sitemap_index.xml

# ================================================
# LEGAL NOTICE - EU DSM DIRECTIVE
# ================================================
# According to Article 4 of EU Directive 2019/790 (DSM Directive)
# we explicitly opt-out from text and data mining (TDM)
# for commercial purposes without license agreement.
#
# TDM-Reservation: 1
# TDM-Policy: https://administrator.de/help/82 (EN: https://administrator.de/help/82?lang=en)
# Terms of Service: https://administrator.de/help/80 (EN: https://administrator.de/help/80?lang=en)
# Privacy Policy: https://administrator.de/help/79 (EN: https://administrator.de/help/79?lang=en)