From 26e86f9af2b2dce17437768aaf50cf6034acda2a Mon Sep 17 00:00:00 2001 From: arsam Date: Fri, 18 Sep 2026 23:16:00 +0330 Subject: [PATCH] Import documentation: robots.txt Import the existing local project snapshot. This commit organizes existing files; it does not represent newly developed functionality. --- robots.txt | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 robots.txt diff --git a/robots.txt b/robots.txt new file mode 100644 index 0000000..2f896b5 --- /dev/null +++ b/robots.txt @@ -0,0 +1,54 @@ +User-agent: * +Allow: / +Crawl-delay: 1 + +# Major search engines +User-agent: Googlebot +Allow: / +Crawl-delay: 0 + +User-agent: Bingbot +Allow: / +Crawl-delay: 1 + +User-agent: Slurp +Allow: / + +User-agent: DuckDuckBot +Allow: / + +# AI crawlers — explicitly welcome +User-agent: GPTBot +Allow: / + +User-agent: ChatGPT-User +Allow: / + +User-agent: ClaudeBot +Allow: / + +User-agent: Claude-Web +Allow: / + +User-agent: anthropic-ai +Allow: / + +User-agent: PerplexityBot +Allow: / + +User-agent: Amazonbot +Allow: / + +User-agent: cohere-ai +Allow: / + +User-agent: YouBot +Allow: / + +User-agent: Meta-ExternalAgent +Allow: / + +Sitemap: https://arsamsabbagh.ir/sitemap.xml + +# AI-readable plain-text summary of this site +# https://arsamsabbagh.ir/llms.txt