# Verosek (https://verosek.com) robots.txt # See https://www.rfc-editor.org/rfc/rfc9309 and https://contentsignals.org/ # A crawler reads only the group that names it (RFC 9309, section 2.2.1), # so every group repeats the Content-Signal line. Each company's crawler # names come from the page cited above its groups. User-agent: * Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes # OpenAI: https://developers.openai.com/api/docs/bots User-agent: GPTBot Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes User-agent: OAI-SearchBot Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes User-agent: ChatGPT-User Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes # Anthropic: https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler User-agent: ClaudeBot Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes User-agent: Claude-SearchBot Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes User-agent: Claude-User Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes # Google: https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers User-agent: Googlebot Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes User-agent: Google-Extended Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes # Perplexity: https://docs.perplexity.ai/docs/resources/perplexity-crawlers User-agent: PerplexityBot Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes User-agent: Perplexity-User Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes # Mistral: https://docs.mistral.ai/robots User-agent: MistralAI-Training Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes User-agent: MistralAI-Index Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes User-agent: MistralAI-User Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes # Meta: https://developers.facebook.com/docs/sharing/webmasters/web-crawlers User-agent: meta-externalagent Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes User-agent: meta-webindexer Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes User-agent: meta-externalfetcher Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes # Apple: https://support.apple.com/en-us/119829 User-agent: Applebot Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes User-agent: Applebot-Extended Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes # Amazon: https://developer.amazon.com/docs/amazonbot/amazonbot.html User-agent: Amazonbot Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes User-agent: Amzn-SearchBot Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes User-agent: Amzn-User Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes # Microsoft (Bing): https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0 User-agent: bingbot Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes # DuckDuckGo: https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot User-agent: DuckDuckBot Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes # DuckDuckGo: https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot User-agent: DuckAssistBot Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes # Common Crawl: https://commoncrawl.org/ccbot User-agent: CCBot Allow: / Content-Signal: search=yes, ai-input=yes, ai-train=yes Sitemap: https://verosek.com/sitemap.xml