#### Robots.txt 
# ========================================================
# This file should not be used to remove pages that are already in google's index.
# In general, only use it for files or pages that search engines should never see, or can significantly impact crawling

## Documentation
# Different search engines interpret directives differently. By default, the first matching directive always wins. But, with Google and Bing, specificity wins.
# Google docs - https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt



### Every bot that might possibly read and respect this file
# ========================================================
User-agent: *

### No SEO value
# Server-side GTM. The web container (GTM-KR7MGSHC) loads from this first-party
# endpoint — /sgtm/gtm.js, /sgtm/ns.html, /sgtm/g/collect — so blocking the prefix
# stops crawlers fetching or executing the tag manager at all. No trailing slash:
# /sgtm/* would leave bare /sgtm and /sgtm?id=... crawlable.
Disallow: /sgtm
Disallow: /*/return/*
Disallow: /*/tripsummary/*
Disallow: /*/checkout/*
Disallow: /*/confirmation/*
Disallow: /api/*

### Query param management
# https://www.peakhour.io/blog/how-to-exclude-query-string-parameters-from-search-engines-using-robots-txt/
Disallow: /*/account/completeResetPassword*?*
Disallow: /*/travel-review*?*
# Next.js RSC payload fetches (?_rsc=... / &_rsc=...). Same content as the page itself,
# never meant to be indexed — crawling them is pure duplicate crawl budget.
Disallow: /*?*_rsc=*
# Marketing/partner attribution params. The nexus middleware moves each into a cookie
# and 302s to the clean URL (wanderu/nexus#888), so crawling these is a redirect to a
# page the crawler already has. aid/tid are anchored on the separator so they cannot
# match ?paid= / ?unpaid=; discountCode is distinctive enough for the loose form.
Disallow: /*?*discountCode=*
Disallow: /*?aid=*
Disallow: /*&aid=*
Disallow: /*?tid=*
Disallow: /*&tid=*


### Old static files that are still live
# Directory where humans upload one-off sitemaps, generally to get google to fix an issue faster
Allow: /guides/sitemap-manual*
Disallow: /guides/

### Cloudflare access protected paths
Disallow: /experiment-override/

### Legacy files/pages that are thought to no longer be live
Disallow: /images/
Disallow: /check/
Disallow: /static/
Disallow: /downtime.php
Disallow: /search/
Disallow: /cheap-unicorn-tickets/
Disallow: /maps/stationMap
Disallow: /popups/
Disallow: /app/css/
Disallow: /fr/billets-de-autobus/
Disallow: /fr/billets-de-train/
Disallow: /fr/stations/
Disallow: /fr/autobuses-et-trains/


### Internet Standards
# https://en.wikipedia.org/wiki/Well-known_URI
Disallow: /.well-known/*
# https://support.google.com/adsense/answer/12171612?hl=en
Disallow: /ads.txt



# Bad or Unhelpful Bots
# ==========================
User-agent: YandexBot
User-agent: Sogou blog
User-agent: Sogou inst spider
User-agent: Sogou News Spider
User-agent: Sogou Orion spider
User-agent: Sogou spider2
User-agent: Sogou web spider
User-agent: AspiegelBot
User-agent: BLEXBot
User-agent: MJ12bot
User-agent: serpstatbot
User-agent: HubSpot Crawler
Disallow: /

# Google-Travel-Flights — partner-feed link verifier. Needs to verify
# deep links into /tripsummary/, which the * group blocks. Allow: / would also
# re-open /sgtm, so the tag manager is re-blocked here (specificity wins for Google).
User-agent: Google-Travel-Flights
Allow: /
Disallow: /sgtm

# AdsBot / AdSense crawlers. Google documents these as ignoring the global group
# outright — "The global user agent (*) is ignored" — so the /sgtm rule in the *
# block never reaches them, and AdsBot renders Ads landing pages for real. Without
# this they load the container and fire GA4 / Google Ads tags on a bot session.
# https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers
# Scoped to the tag manager only: landing pages stay crawlable so ad quality and
# destination checks keep working.
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
Disallow: /sgtm

# Independent of user agent. Links in the sitemap are full URLs using https:// and need to match
# the protocol of the sitemap.
Sitemap: https://www.wanderu.com/sitemap.xml