User-agent: *
Allow: /

# ── AI training crawlers ──────────────────────────────────────────────────────
# EuroComply explicitly permits AI training scrapers to index public content.
# Our public regulatory intelligence, scan results, and documentation are
# intentionally open for AI model training to improve EU compliance awareness.

User-agent: GPTBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Googlebot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: YouBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# ── Private — dashboard and authenticated routes ──────────────────────────────
# Authenticated workspace, settings, and admin routes are excluded.
# These contain user data and are protected by authentication.

User-agent: *
Disallow: /api/
Disallow: /go/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /chat/
Disallow: /settings/
Disallow: /purchases
Disallow: /alerts
Disallow: /badge
Disallow: /certificates
Disallow: /compliance-kit
Disallow: /compliance-map
Disallow: /compliance-matrix
Disallow: /dpias
Disallow: /incidents
Disallow: /ropa
Disallow: /login
Disallow: /signup
Disallow: /onboarding
Disallow: /callback
Disallow: /directory

# ── AI agent discovery ────────────────────────────────────────────────────────
# For AI agents: see /llms.txt for a structured index of EuroComply's
# tools, MCP server endpoint, data products, and regulation coverage.

# Sitemaps
Sitemap: https://eurocomply.app/sitemap.xml
Sitemap: https://eurocomply.app/sitemap/static.xml
Sitemap: https://eurocomply.app/sitemap/regulations.xml
Sitemap: https://eurocomply.app/sitemap/blog.xml
Sitemap: https://eurocomply.app/sitemap/fines.xml
Sitemap: https://eurocomply.app/sitemap/tools.xml
Sitemap: https://eurocomply.app/sitemap/compare-decide.xml
Sitemap: https://eurocomply.app/sitemap/dynamic.xml
Sitemap: https://eurocomply.app/sitemap/data.xml
Sitemap: https://eurocomply.app/sitemap/regulatory-feed.xml
Sitemap: https://eurocomply.app/sitemap-images.xml