# https://vdf.ai/ — robots.txt
# Canonical host is https://vdf.ai (apex, HTTPS). www and http variants 301 to apex.

User-agent: *
Allow: /

# Block server endpoints, drafts, and private areas
Disallow: /v1/
Disallow: /admin/
Disallow: /private/

# Block app, auth, and account-state URLs (not SEO landing pages)
Disallow: /consultant/
Disallow: /login
Disallow: /register/
Disallow: /payment/
Disallow: /account/

# Faceted blog query URLs redirect to the clean path at nginx. Keep them
# crawlable while Google processes those redirects and consolidates old URLs.

# Allow common SEO assets explicitly (some crawlers need this)
Allow: /assets/
Allow: /_assets/
Allow: /images/
Allow: /favicon.ico
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /rss.xml
Allow: /956d2db2fad34cd8bb35e669b96032c2.txt

# Aggressive scrapers that honour robots.txt and send no referral traffic.
# NOTE: no explicit Allow group for Googlebot / GPTBot / ClaudeBot / PerplexityBot /
# Bingbot. A crawler obeys only its most specific group, so naming them here would
# drop them out of the User-agent: * group above and re-open /v1/, /admin/ and the
# account paths to them. They are already allowed, and they feed the AI-assistant
# channel that produces roughly half of all key events — never block them.
User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

# Sitemap
Sitemap: https://vdf.ai/sitemap.xml

# AI discovery: see https://vdf.ai/llms.txt for a structured content summary
# and https://vdf.ai/llms-full.txt for the complete machine-readable index.