diff --git a/robots.txt b/robots.txt new file mode 100644 index 0000000..2f896b5 --- /dev/null +++ b/robots.txt @@ -0,0 +1,54 @@ +User-agent: * +Allow: / +Crawl-delay: 1 + +# Major search engines +User-agent: Googlebot +Allow: / +Crawl-delay: 0 + +User-agent: Bingbot +Allow: / +Crawl-delay: 1 + +User-agent: Slurp +Allow: / + +User-agent: DuckDuckBot +Allow: / + +# AI crawlers — explicitly welcome +User-agent: GPTBot +Allow: / + +User-agent: ChatGPT-User +Allow: / + +User-agent: ClaudeBot +Allow: / + +User-agent: Claude-Web +Allow: / + +User-agent: anthropic-ai +Allow: / + +User-agent: PerplexityBot +Allow: / + +User-agent: Amazonbot +Allow: / + +User-agent: cohere-ai +Allow: / + +User-agent: YouBot +Allow: / + +User-agent: Meta-ExternalAgent +Allow: / + +Sitemap: https://arsamsabbagh.ir/sitemap.xml + +# AI-readable plain-text summary of this site +# https://arsamsabbagh.ir/llms.txt