# robots.txt for linkdevelopment.com
# Last-Updated: October 20, 2025
# Maintained by: Marketing Department

# ================================
# ALLOW SEARCH & DISCOVERY BOTS
# ================================

# OpenAI ChatGPT Search
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic (Claude) Search
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Perplexity Search
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google Generative / Search
User-agent: Googlebot
Allow: /

User-agent: Google-Extended
Allow: /

# Bing and Common Crawlers
User-agent: Bingbot
Allow: /

# ================================
# BLOCK AI TRAINING & SCRAPING BOTS
# ================================

# OpenAI GPTBot (model training)
User-agent: GPTBot
Disallow: /

# Common Crawl (used for AI datasets)
User-agent: CCBot
Disallow: /

# Facebook Research / AI
User-agent: FacebookBot
Disallow: /

# Amazon Bot (AI data collection)
User-agent: Amazonbot
Disallow: /

# Applebot (used in AI training)
User-agent: Applebot
Disallow: /

# Bytespider (ByteDance / TikTok AI)
User-agent: Bytespider
Disallow: /

# You.com (AI assistant / training)
User-agent: YouBot
Disallow: /

# CoCounsel / Casetext (Legal AI)
User-agent: CasetextBot
Disallow: /

# Kagi Search AI
User-agent: KagiBot
Disallow: /

# Neeva (AI search / training)
User-agent: NeevaAI
Disallow: /

# Diffbot (data extraction for AI)
User-agent: Diffbot
Disallow: /

# Scrapy and Generic Crawlers
User-agent: Scrapy
Disallow: /

# DataForSeo / AI Data Agents
User-agent: DataForSeoBot
Disallow: /

# ================================
# DEFAULT RULES FOR ALL OTHER BOTS
# ================================

# Allow all other bots by default, but block wp-admin
User-agent: *
Disallow: /wp/wp-admin/
Allow: /wp-admin/admin-ajax.php

# ================================
# SITEMAP
# ================================
Sitemap: https://linkdevelopment.com/sitemap_index.xml