User-agent: *
Disallow: /wp/wp-content/uploads/wc-logs/
Disallow: /wp/wp-content/uploads/woocommerce_transient_files/
Disallow: /wp/wp-content/uploads/woocommerce_uploads/
Disallow: /*?add-to-cart=
Disallow: /*?*add-to-cart=
Disallow: /wp/wp-admin/
Allow: /wp/wp-admin/admin-ajax.php

Sitemap: https://dcrand.org/wp-sitemap.xml

#############################################
# randonneur.app Network - robots.txt
# Site: dcrand.org
# Generated dynamically
# Last updated: January 2026
#############################################

#############################################
# MAJOR SEARCH ENGINES (Welcome!)
#############################################

# Google - Primary search
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Googlebot-Video
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /login/
Disallow: /members/
Disallow: /groups/
Disallow: /activity/
Disallow: /forums/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /pre-register/
Disallow: /register/
Disallow: /confirm-device/
Disallow: /?s=
Disallow: /search/
Disallow: /*?add-to-cart=*
Disallow: /*?replytocom=*
Disallow: /wp-json/
Disallow: /feed/
Disallow: /comments/feed/
Crawl-delay: 1

# Bing - Microsoft search
User-agent: Bingbot
User-agent: MSNBot
User-agent: MSNBot-Media
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /login/
Disallow: /members/
Disallow: /groups/
Disallow: /activity/
Disallow: /forums/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /pre-register/
Disallow: /register/
Disallow: /confirm-device/
Disallow: /?s=
Disallow: /search/
Disallow: /*?add-to-cart=*
Crawl-delay: 1

# DuckDuckGo - Privacy-focused search
User-agent: DuckDuckBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /login/
Disallow: /members/
Disallow: /groups/
Disallow: /activity/
Disallow: /forums/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /pre-register/
Disallow: /register/
Disallow: /confirm-device/
Disallow: /?s=
Disallow: /search/

# Yahoo (uses Bing infrastructure)
User-agent: Slurp
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /login/
Disallow: /members/
Disallow: /groups/
Disallow: /activity/
Disallow: /forums/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /pre-register/
Disallow: /register/
Disallow: /confirm-device/
Crawl-delay: 1

# Apple - Siri and Spotlight
User-agent: Applebot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /login/
Disallow: /members/
Disallow: /groups/
Disallow: /activity/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/

# Yandex - Russian search (some US users)
User-agent: Yandex
User-agent: YandexBot
User-agent: YandexImages
Allow: /
Disallow: /wp-admin/
Disallow: /members/
Disallow: /my-account/
Disallow: /cart/
Disallow: /checkout/
Crawl-delay: 2

# Brave Search
User-agent: Brave
Allow: /
Disallow: /wp-admin/
Disallow: /members/
Disallow: /my-account/
Disallow: /cart/
Disallow: /checkout/

#############################################
# AI CRAWLERS - BLOCK TRAINING
# These scrape content for AI model training
# Block to protect member content and IP
#############################################

# OpenAI - GPT model training
User-agent: GPTBot
Disallow: /

# Google - Gemini/Bard AI training (separate from search)
User-agent: Google-Extended
Disallow: /

# Common Crawl - Dataset used by many AI companies
User-agent: CCBot
Disallow: /

# Anthropic - Claude AI training
User-agent: anthropic-ai
User-agent: ClaudeBot
Disallow: /

# Meta - Facebook/Instagram AI training
User-agent: FacebookBot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
Disallow: /

# ByteDance - TikTok AI
User-agent: Bytespider
Disallow: /

# Amazon - Alexa and AWS AI
User-agent: Amazonbot
Disallow: /

# Cohere - Enterprise AI
User-agent: cohere-ai
Disallow: /

# Diffbot - Web scraping AI
User-agent: Diffbot
Disallow: /

# AI2 - Allen Institute
User-agent: AI2Bot
Disallow: /

# Hugging Face
User-agent: HuggingFaceBot
Disallow: /

# Writer.com
User-agent: WriterBot
Disallow: /

# Webz.io data collector
User-agent: Webzio
User-agent: webzio-extended
Disallow: /

# Siri suggestions (training, not search)
User-agent: Applebot-Extended
Disallow: /

#############################################
# AI CRAWLERS - ALLOW SEARCH/BROWSING
# These provide real-time answers citing sources
# Allow so randonneuring info appears in AI search
#############################################

# ChatGPT browsing - Real-time lookups when users ask questions
User-agent: ChatGPT-User
Allow: /
Disallow: /wp-admin/
Disallow: /members/
Disallow: /my-account/
Disallow: /cart/
Disallow: /checkout/
Disallow: /pre-register/
Disallow: /register/

# Perplexity AI - AI-powered search engine
User-agent: PerplexityBot
Allow: /
Disallow: /wp-admin/
Disallow: /members/
Disallow: /my-account/
Disallow: /cart/
Disallow: /checkout/
Disallow: /pre-register/
Disallow: /register/

# You.com - AI search
User-agent: YouBot
Allow: /
Disallow: /wp-admin/
Disallow: /members/
Disallow: /my-account/
Disallow: /cart/
Disallow: /checkout/

# Neeva (now part of Snowflake, but bot may still exist)
User-agent: NeevaBot
Allow: /
Disallow: /wp-admin/
Disallow: /members/
Disallow: /my-account/

#############################################
# SEO TOOL BOTS - BLOCK
# Competitor intelligence tools
# No benefit, wastes bandwidth
#############################################

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: BomboraBot
Disallow: /

#############################################
# AGGRESSIVE/MALICIOUS CRAWLERS - BLOCK
#############################################

User-agent: MegaIndex
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: Riddler
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: Linguee
Disallow: /

User-agent: DataForSeoBot
Disallow: /

#############################################
# DEFAULT RULES (All other crawlers)
#############################################

User-agent: *
Allow: /

# WordPress core - protect admin
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Allow: /wp-admin/admin-ajax.php

# Authentication & user flows
Disallow: /login/
Disallow: /logout/
Disallow: /register/
Disallow: /pre-register/
Disallow: /confirm-device/
Disallow: /password-reset/
Disallow: /activate/

# BuddyBoss member areas - private content
Disallow: /members/
Disallow: /groups/
Disallow: /activity/
Disallow: /forums/
Disallow: /notifications/
Disallow: /messages/
Disallow: /settings/

# WooCommerce - transactional pages
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /order-received/
Disallow: /*?add-to-cart=*
Disallow: /*?removed_item=*

# Event Tickets - personal ticket pages
Disallow: /*?ical=*
Disallow: /tickets/
Disallow: /ticket/

# Cycling Club Manager - ride management
Disallow: /completed-rides/
Disallow: /my-rides/

# Search & dynamic content
Disallow: /?s=
Disallow: /search/
Disallow: /*?replytocom=*
Disallow: /*?p=*

# Pagination - reduce duplicate content
Disallow: /page/

# Feeds - prevent duplicate content indexing
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /*/feed/

# Trackbacks - spam vector
Disallow: /trackback/
Disallow: /*/trackback/

# Caching/optimization artifacts
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/cache/
Disallow: /wp-content/litespeed/

# Misc WordPress
Disallow: /readme.html
Disallow: /license.txt
Disallow: /wp-config.php

#############################################
# CRAWL RATE
# Be gentle - volunteer-run infrastructure
#############################################

Crawl-delay: 2