# robots.txt for https://gta6intelhub.com
# Physical file served from the site root. Verify after deploy:
#   curl -s https://gta6intelhub.com/robots.txt
#
# ---------------------------------------------------------------------------
# AI CRAWLER POLICY — a deliberate decision, documented so it can be reversed.
#
# This site's entire reason to exist is to be the source that gets cited when
# someone asks "when does GTA 6 come out?" or "is GTA 6 delayed?" — in Google,
# in ChatGPT, in Copilot, in Perplexity, in Claude, in Gemini. Those assistants
# only cite what they are allowed to fetch. Blocking them does not protect the
# content (the facts are public record and every other GTA site publishes them);
# it just removes us from the answer and hands the citation to a competitor.
#
# We also have nothing to withhold. There is no paywall, no proprietary dataset,
# no licensed art. The value here is the confidence-tier labelling and the
# sourcing discipline — which only pays off if it is seen and attributed.
#
# So: ALL major AI crawlers are allowed below, including the pure-training ones
# (GPTBot, CCBot, Google-Extended). Training exposure is how a brand becomes a
# default answer without a live fetch, which is the long game for a reference
# site.
#
# TO REVERSE: change `Allow: /` to `Disallow: /` under the specific bot.
#   - Blocking GPTBot / CCBot / Google-Extended stops TRAINING use but keeps
#     live-citation ability (those are separate bots — see the next group).
#   - Blocking OAI-SearchBot / ChatGPT-User / PerplexityBot / ClaudeBot /
#     Claude-SearchBot removes this site from AI ANSWERS entirely. Do not do
#     that unless the strategy has fundamentally changed.
#   - Google-Extended does NOT affect Google Search ranking either way.
# ---------------------------------------------------------------------------

# ---- Default: every standard crawler, full access ----
User-agent: *
Allow: /
# Client-side search results are thin, infinite and duplicate the databases.
Disallow: /search
Disallow: /*?q=
# Never block CSS/JS/fonts — Google needs them to render the page.
Allow: /_astro/
Allow: /pagefind/

# ---- AI search / answer crawlers: ALLOW (these are the ones that cite us) ----
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: meta-externalagent
Allow: /

# ---- AI training crawlers: ALLOW (see rationale above) ----
User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

# ---- Search engines: explicit, for clarity ----
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: YandexBot
Allow: /

# ---- Aggressive scrapers with no citation upside: block ----
# These take bandwidth and give nothing back — no referral, no attribution.
User-agent: Bytespider
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

# ---- SEO tool crawlers: block (they only feed competitor research) ----
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# ---- Sitemap (generated by @astrojs/sitemap at build time) ----
Sitemap: https://gta6intelhub.com/sitemap-index.xml

# ---- AI-agent brief ----
# Curated, machine-readable summary of this site's confirmed facts, in the
# llms.txt format (https://llmstxt.org):
#   https://gta6intelhub.com/llms.txt
#
# Full content — every guide, news article and database record, with its
# confidence tier, sources and last-verified date. Generated from the site's
# own records at build time, so it cannot go stale:
#   https://gta6intelhub.com/llms-full.txt
#
# Structured feeds, if you would rather parse than scrape. The directory lists
# all of them with record counts and the tier definitions:
#   https://gta6intelhub.com/feeds.json