# robots.txt for melashri.net
#
# Policy:
# - Allow ordinary search engines.
# - Allow AI search, retrieval, and user-triggered summarization.
# - Disallow use for model training where crawlers respect robots.txt.
#
# Content-Signal meanings:
# search=yes    allow classic search indexing and search results
# ai-input=yes  allow retrieval, grounding, and agent/user-triggered summaries
# ai-train=no   do not use content for model training or fine-tuning

User-agent: *
# Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

# OpenAI training crawler.
# OAI-SearchBot and ChatGPT-User remain allowed by the wildcard rule.
User-agent: GPTBot
Disallow: /

# Google AI training/control token.
# Googlebot remains allowed by the wildcard rule.
User-agent: Google-Extended
Disallow: /

# Apple generative AI training control token.
# Applebot search remains allowed by the wildcard rule.
User-agent: Applebot-Extended
Disallow: /

# Anthropic training crawler.
# Any Claude search/user fetchers should remain allowed by the wildcard rule.
User-agent: ClaudeBot
Disallow: /

# Common Crawl corpus crawler.
User-agent: CCBot
Disallow: /

# Meta AI crawler.
User-agent: meta-externalagent
Disallow: /

# ByteDance AI crawler.
User-agent: Bytespider
Disallow: /

Sitemap: https://melashri.net/sitemap.xml