# This file is served by the app at /dutch-tax/robots.txt (its basePath) — the true domain
# root https://nltaxcalculator.nl/robots.txt is what actually governs crawler/bot access, and is
# served by an nginx proxy rule (added 2026-09-19, /etc/nginx/conf.d/nltaxcalculator.conf,
# VM-only, not tracked in git) that forwards it straight to this file, same pattern as the
# existing /sitemap.xml proxy below. A prior version of this comment claimed Cloudflare's "AI
# Crawl Control" auto-generated a root robots.txt of its own — confirmed false on 2026-09-19
# (Google Search Console showed the root 404ing, and a direct curl confirmed no Cloudflare-
# injected file existed).
User-agent: *
Disallow: /api/
Disallow: /admin
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /auth/
# The app lives under the /dutch-tax basePath, so the rules above (domain-root paths) never
# matched its real URLs; these do. Keeps crawl budget on content pages.
Disallow: /dutch-tax/api/
Disallow: /dutch-tax/admin
Disallow: /dutch-tax/login
Disallow: /dutch-tax/register
Disallow: /dutch-tax/forgot-password
Disallow: /dutch-tax/reset-password
Disallow: /dutch-tax/auth/
# Read-only endpoints public pages render from (forum, news, flags, benchmark stats): Google
# must be able to fetch these to see that content. The longer Allow wins over Disallow above.
Allow: /dutch-tax/api/forum/
Allow: /dutch-tax/api/news
Allow: /dutch-tax/api/feature-flags
Allow: /dutch-tax/api/benchmarks/stats
# Ad images Instagram downloads while publishing (lib/instagram.ts).
Allow: /dutch-tax/api/instagram-media/
Allow: /

Sitemap: https://nltaxcalculator.nl/sitemap.xml