# gracker.ai — robots.txt
# Last reviewed: 2026-09-22
#
# Policy: the whole public site is open to every crawler, including every AI
# answer engine. Only Cloudflare internals and the JSON API are excluded.
#
# IMPORTANT for anyone editing this file: a named "User-agent:" group OVERRIDES
# the "User-agent: *" group entirely for that agent — rules omitted from a named
# group are NOT inherited. The AI group below therefore repeats the wildcard
# rules verbatim. If you change one group, change the other.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /cdn-cgi/
Disallow: /api/
Allow: /

# --- AI answer engines, AI search indexes and AI crawlers: explicitly allowed.
# Identical rules to the wildcard group above. Tokens taken from each operator's
# own crawler documentation where one exists:
#   OpenAI      https://developers.openai.com/api/docs/bots
#   Anthropic   https://support.claude.com/en/articles/8896518
#   Perplexity  https://docs.perplexity.ai/guides/bots
#   Google      https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers
#   Apple       https://support.apple.com/en-us/119829
#   Meta        https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/
#   Mistral     https://docs.mistral.ai/robots
#   Microsoft   https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0
# Remaining tokens are from the community registry
# https://github.com/ai-robots-txt/ai.robots.txt (robots.json).
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: OAI-AdsBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: anthropic-ai
User-agent: Googlebot
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Google-CloudVertexBot
User-agent: bingbot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: meta-webindexer
User-agent: facebookexternalhit
User-agent: FacebookBot
User-agent: MistralAI-User
User-agent: MistralAI-Index
User-agent: MistralAI-Training
User-agent: Bytespider
User-agent: TikTokSpider
User-agent: CCBot
User-agent: cohere-ai
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: Amazonbot
User-agent: PetalBot
User-agent: Diffbot
User-agent: Timpibot
User-agent: Omgilibot
User-agent: Webzio-Extended
User-agent: ImagesiftBot
User-agent: AI2Bot
User-agent: Andibot
User-agent: SemrushBot-OCOB
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /cdn-cgi/
Disallow: /api/
Allow: /

# Sitemaps. This site's own sitemap is the INDEX (sitemap-index.xml), which
# already contains sitemap-0.xml plus the cybersecurity-tools and
# brand-ai-visibility shards — do not list shards here as well.
# The remaining entries are separately hosted programmatic portals.
Sitemap: https://gracker.ai/sitemap-index.xml
Sitemap: https://gracker.ai/blog/sitemap.xml
Sitemap: https://gracker.ai/docs/sitemap.xml
Sitemap: https://gracker.ai/news/sitemap.xml
Sitemap: https://gracker.ai/stories/sitemap.xml
Sitemap: https://gracker.ai/events/sitemap-index.xml
Sitemap: https://gracker.ai/questions/sitemap.xml
Sitemap: https://gracker.ai/glossary/sitemap.xml
Sitemap: https://gracker.ai/seo-glossary/sitemap.xml
Sitemap: https://gracker.ai/cyber-resources/sitemap.xml
Sitemap: https://gracker.ai/b2b-saas-growth-tools/sitemap.xml
Sitemap: https://gracker.ai/programmatic-seo-101/sitemap.xml
Sitemap: https://gracker.ai/cybersecurity-marketing-101/sitemap.xml
Sitemap: https://gracker.ai/seo-101/sitemap.xml
Sitemap: https://gracker.ai/dmg/sitemap.xml