User-agent: *
Disallow:
Disallow: /cgi-bin/
# The admin is login-gated, so nothing under it was ever indexable - but saying
# so stops compliant crawlers walking admin URLs they picked up elsewhere (a
# pasted link, an old referrer) and tripping the AI guard on the way past.
Disallow: /admin
# Honeypot: linked only from a hidden, aria-hidden block. A compliant crawler
# never requests it, so a hit means the page was read programmatically.
Disallow: /aitrap

# AI TRAINING crawlers only. These collect pages to train models and send
# nothing back - no visits, no citations, no customers - so refusing them costs
# nothing. Search and answer-engine crawlers (OAI-SearchBot, PerplexityBot,
# Applebot) are deliberately NOT listed: they decide whether this store appears
# when someone asks an assistant for meal prep nearby, and the storefront is
# public anyway. They are blocked on /admin by the User-Agent gate instead.
#
# Google-Extended and Applebot-Extended only exist here - they are robots.txt
# tokens, never User-Agents. They opt out of Gemini and Apple Intelligence
# training without affecting Google Search or Siri.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: CCBot
User-agent: Bytespider
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: AI2Bot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Omgilibot
User-agent: Webzio-Extended
User-agent: Timpibot
User-agent: Kangaroo Bot
User-agent: PanguBot
User-agent: Poseidon Research Crawler
User-agent: TikTokSpider
User-agent: SemrushBot-OCOB
User-agent: DataForSeoBot
Disallow: /