# robots.txt — eSIM.site
# Public catalogue (destinations, regions, packages, blog, guides, FAQ) is fully
# crawlable. Private, transactional, and infinite-parameter areas are blocked to
# preserve crawl budget and keep user/order data out of the index.

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /includes/
Disallow: /uploads/kyc/
Disallow: /cache/
Disallow: /account/
Disallow: /checkout
Disallow: /order-processing
Disallow: /order-confirmation
Disallow: /search
# Auth-flow and upsell endpoints are POST/JS surfaces that 404 on a plain GET —
# crawling them only produces "Not found (404)" errors in Search Console.
Disallow: /auth/
Disallow: /upsell/
# Sign-in URLs with a return address (/login?redirect=...) used 380+ Search
# Console crawl slots. Only the ?-variants are blocked: /login itself stays
# crawlable so Google can still read its noindex.
Disallow: /login?
Disallow: /signup?
Disallow: /register?
# Block raw *.php URLs (duplicates of the pretty-URL pages). Only matches URLs
# ENDING in .php, so pretty paths and /sitemap*.xml are unaffected.
Disallow: /*.php$
Allow: /sitemap.xml
Allow: /sitemap-*.xml

# ---------------------------------------------------------------------------
# AI answer engines / LLM crawlers.
# We WANT to be cited in AI answers ("best eSIM for Japan", pricing, coverage),
# so the search-and-cite bots are welcomed on the public catalogue. A named
# user-agent group fully overrides the "*" group above, so the private-area
# disallows are repeated in each. Keep these in sync with the "*" block.
# ---------------------------------------------------------------------------
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /includes/
Disallow: /uploads/kyc/
Disallow: /cache/
Disallow: /account/
Disallow: /checkout
Disallow: /order-processing
Disallow: /order-confirmation
Disallow: /search
Disallow: /auth/
Disallow: /upsell/
Disallow: /login?
Disallow: /signup?
Disallow: /register?
Disallow: /*.php$
Allow: /sitemap.xml
Allow: /sitemap-*.xml

# Yandex: drop tracking parameters so tagged links are not crawled as
# separate pages. Clean-param is cross-section, so it applies to every group.
Clean-param: utm_source&utm_medium&utm_campaign&utm_content&utm_term&gclid&fbclid&ttclid&msclkid&yclid

# Sitemap
Sitemap: https://esim.site/sitemap.xml