User-agent: *
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /admin/
Disallow: /private/
Disallow: /tmp/
Sitemap: https://dashboard.llmtraining.dev/sitemap.xml

# Priority crawl paths
Allow: /transformer-stack
Allow: /fine-tuning
Allow: /visualisations
Allow: /telemetry
Allow: /huggingface-sync