User-agent: *
Allow: /

Sitemap: https://stt.ai/sitemap.xml

# Internal/admin paths
Disallow: /admin/
Disallow: /api/internal/
Disallow: /ipns/
Disallow: /widget/embed/

# UI-state query params — `/transcribe/?language=X&model=Y` style internal
# links produce duplicate-content variants of the bare path with identical
# body. Block crawling at the source so Google never spends budget on
# them. The standard list per ~/seo/playbook/06-robots-and-sitemap.md;
# pre-emptive even though stt.ai's current sitemap is clean (sister
# project tts.ai got SpamBrained 2026-04-27 for exactly this shape via
# the `VoiceModelSitemap` + `?model=<slug>` pre-select URLs).
Disallow: /*?ref=
Disallow: /*&ref=
Disallow: /*?from=
Disallow: /*&from=
Disallow: /*?next=
Disallow: /*&next=
Disallow: /*?return=
Disallow: /*&return=
Disallow: /*?sort=
Disallow: /*&sort=
Disallow: /*?new=
Disallow: /*&new=
Disallow: /*?is_free=
Disallow: /*&is_free=
Disallow: /*?tool=
Disallow: /*&tool=
Disallow: /*?model=
Disallow: /*&model=
Disallow: /*?language=
Disallow: /*&language=
Disallow: /*?voice=
Disallow: /*&voice=
Disallow: /*?gender=
Disallow: /*&gender=

# /pricing/ became a TAB ROW on 2026-09-22 (one panel at a time, chosen by a
# real server-resolved <a href> so it works without JavaScript). That is the
# right build, but it turned one page into `view` x `plan` x `billing` x every
# indexable `lang` -- roughly 3,500 URLs that all render the same SKU set. The
# canonical already folds them to /pricing/?lang=xx, so there is no index-bloat
# risk; the cost is pure crawl budget, which this file already protects against
# for ten other params. Measured 2026-09-23, the first full day after the
# change: 1,296 /pricing/ fetches in 18 h against a 20-98/day baseline, almost
# all of them `?lang=X&plan=Y&view=Z` permutations.
Disallow: /*?view=
Disallow: /*&view=
Disallow: /*?billing=
Disallow: /*&billing=
Disallow: /*?plan=
Disallow: /*&plan=

# `?lang=en` is the default with a redundant param. Server-side
# StripUnsupportedLangMiddleware also 301s these; the Disallow stops
# Google from spending crawl on them. Other `?lang=<iso>` values are
# intentionally NOT blocked — indexable languages need ?lang=xx URLs
# crawlable as proper alternates via hreflang.
Disallow: /*?lang=en

# Clicky first-party analytics paths (anti-adblock, proxied on our domain).
# Googlebot re-crawls the same-origin beacon (/4e2503b3d5e?site_id=...) + loader as
# junk 200s — pure crawl-budget waste (~49% of refresh re-crawls on free.ai).
# robots.txt only governs crawlers; browsers still run analytics. See ~/seo/playbook/06.
Disallow: /4e2503b3d5e
Disallow: /1c3d3c08227.js