# /redirect is deliberately NOT disallowed for the indexing crawlers, even
# though it is not real content. It was disallowed, and that turned out to be
# the thing preventing cleanup: ~1.14M discovered spam URLs sat permanently
# "pending" in Search Console's validation because Googlebot was forbidden
# from fetching them and so could never learn they are gone. Blocking a URL
# does not remove it from the index; it only stops Google learning anything
# new about it. These now answer 410 (see the /redirect branch in
# src/middleware.ts), and a crawler must be allowed to fetch that 410 for the
# URL to drop out. Re-adding a Disallow here would freeze them in the index.
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Twitterbot
Allow: /
Disallow: /redirect

User-agent: facebookexternalhit
Allow: /
Disallow: /redirect

# Explicit allow rules for AI-answer-engine / AI-training crawlers, for
# clarity and intentionality. These do not change actual access: the
# wildcard `User-agent: * / Allow: /` block below already permits every
# crawler, including the ones named here.
User-agent: GPTBot
Allow: /
Disallow: /redirect

User-agent: ChatGPT-User
Allow: /
Disallow: /redirect

User-agent: ClaudeBot
Allow: /
Disallow: /redirect

User-agent: Claude-Web
Allow: /
Disallow: /redirect

User-agent: PerplexityBot
Allow: /
Disallow: /redirect

User-agent: Google-Extended
Allow: /
Disallow: /redirect

User-agent: CCBot
Allow: /
Disallow: /redirect

User-agent: Amazonbot
Allow: /
Disallow: /redirect

User-agent: Bytespider
Allow: /
Disallow: /redirect

# /redirect is a Facebook-click-ID attribution "smart link" page, not real
# content — and was found to be the target of an actively-exploited,
# large-scale (1.14M+ URL) spam-indexing pattern (see
# src/app/(pages)/redirect/page.tsx and src/utils/redirectAllowlist.ts for
# the underlying fix). It is left crawlable here for the reason given at the
# top of this file: the spam URLs answer 410, and Google has to be able to
# fetch that 410 to drop them. The named AI-crawler and social-preview
# groups above keep their Disallow — they do not maintain a search index, so
# there is nothing to remove and no reason to spend their crawl on it.
User-agent: *
Allow: /

Sitemap: https://tickit.co/sitemap.xml