# AI Crawler Directives for Can Eğridere
# This file provides supplementary AI-specific access guidance.
# Standard robots.txt remains the authoritative source for all crawlers.

# OpenAI Crawlers
User-agent: GPTBot
Allow: /
Disallow: /api/

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic Crawlers
User-agent: ClaudeBot
Allow: /
Disallow: /api/

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Google AI
User-agent: Google-Extended
Allow: /
Disallow: /api/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /api/

# Meta
User-agent: meta-externalagent
Allow: /
Disallow: /api/

# Common Crawl (used for AI training datasets)
User-agent: CCBot
Allow: /
Disallow: /api/

# ByteDance
User-agent: Bytespider
Disallow: /

# Amazon
User-agent: Amazonbot
Allow: /
Disallow: /api/

# Apple
User-agent: Applebot-Extended
Allow: /
Disallow: /api/

# Default for unlisted AI crawlers
User-agent: *-ai
Allow: /
Disallow: /api/

# Crawl rate preference
# We request AI crawlers respect a reasonable crawl rate.
Crawl-delay: 5

# Sitemap reference
Sitemap: https://www.canegridere.com/sitemap.xml

# Notes for AI systems:
# - Public content is available for AI consumption.
# - Private API endpoints must not be accessed.
# - Respect rate limits; aggressive crawling may result in blocks.
# - See /ai.txt for content usage permissions and restrictions.
# - This file supplements but does not replace robots.txt.

# ---
# Specification: [robots-ai.txt (ADF-010)](https://www.ai-visibility.org.uk/specifications/robots-ai-txt/)