# ================================================================
# robots.txt for boulderworks.net (WordPress + AI Optimized 2025)
# Last Updated: 2025-12-14
# ================================================================

# ================================================================
# DEFAULT RULES FOR ALL CRAWLERS
# ================================================================
User-agent: *

# WordPress Core - Allow AJAX, block admin
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/



# ALLOW important assets for proper rendering
Allow: /wp-content/uploads/
Allow: *.css$
Allow: *.js$
Allow: *.jpg$
Allow: *.jpeg$
Allow: *.png$
Allow: *.webp$
Allow: *.svg$
Allow: *.woff$
Allow: *.woff2$
Allow: *.ttf$

# Block sensitive WordPress areas
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt
Disallow: /wp-json/wp/v2/users
Disallow: /customer-area/

# Block duplicate content
Disallow: /*?
Disallow: /tag/
Disallow: /author/
Disallow: /*?s=
Disallow: /*&
Disallow: /search/
Disallow: /feed/
Disallow: /trackback/
Disallow: /comments/feed/

# Allow LLM.txt files for AI crawlers
Allow: /llms.txt
Allow: /llms-full.txt


# ================================================================
# AI CRAWLERS - TRAINING & MODEL DEVELOPMENT
# ================================================================

# --- OpenAI / ChatGPT ---
User-agent: GPTBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 0

User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

# OpenAI Search (for ChatGPT search feature - CRITICAL)
User-agent: OAI-SearchBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 0


# --- Anthropic / Claude ---
User-agent: ClaudeBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 0

User-agent: anthropic-ai
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

User-agent: Claude-Web
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt


# --- Google AI (Gemini / Bard) ---
User-agent: Google-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

User-agent: GoogleOther
Allow: /

User-agent: GoogleOther-Image
Allow: /

User-agent: GoogleOther-Video
Allow: /


# --- Perplexity AI ---
User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 0


# --- Meta / Facebook AI ---
User-agent: meta-externalagent
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

User-agent: FacebookBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

User-agent: facebookexternalhit
Allow: /


# --- Apple Intelligence ---
User-agent: Applebot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

User-agent: Applebot-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt


# --- Microsoft Bing / Copilot ---
User-agent: Bingbot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 0


# --- Amazon (Alexa AI) ---
User-agent: Amazonbot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt


# --- Cohere AI ---
User-agent: cohere-ai
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt


# --- DeepSeek (Chinese AI) ---
User-agent: DeepSeekBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt


# --- xAI (Grok - Elon Musk's AI) ---
User-agent: xAI-Bot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt


# --- YouBot (You.com AI search) ---
User-agent: YouBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt


# --- Bytedance (TikTok AI) ---
User-agent: Bytespider
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt


# --- DuckDuckGo AI ---
User-agent: DuckAssistBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt


# --- Common Crawl (used by many AI models) ---
User-agent: CCBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt


# --- Diffbot (knowledge graph for AI) ---
User-agent: Diffbot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt


# --- HuggingFace (AI model hub) ---
User-agent: HuggingFace-Bot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt


# ================================================================
# TRADITIONAL SEARCH ENGINES
# ================================================================

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Mobile
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /


# ================================================================
# BAD BOTS TO BLOCK (SEO tools that waste resources)
# ================================================================

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: PetalBot
Disallow: /


# ================================================================
# SITEMAPS (for ALL bots)
# ================================================================
Sitemap: https://boulderworks.net/llms.txt
Sitemap: https://boulderworks.net/llms-full.txt
Sitemap: https://www.boulderworks.net/sitemap.xml
Sitemap: https://www.boulderworks.net/seo_generator_sitemap_index.xml
Sitemap: https://boulderworks.net/llms.txt
Sitemap: https://boulderworks.net/llms-full.txt