# robots.txt for https://clrt.so

# Allow all crawlers
User-agent: *
Allow: /
Allow: /portal
Allow: /guides
Allow: /guides/search
Allow: /legal/files/privacy
Allow: /legal/files/tos
Allow: /auth

# Temporary files and development paths
Disallow: /tmp/
Disallow: /dev/
Disallow: /test/
Disallow: /.svelte-kit/
Disallow: /node_modules/

# API endpoints
Disallow: /api/

# Sitemap location
Sitemap: https://clrt.so/sitemap.xml

# Crawl-delay for heavy crawlers
Crawl-delay: 1

# Additional SEO optimization rules
Allow: /*.js
Allow: /*.css
Allow: /*.png
Allow: /*.jpg
Allow: /*.gif
Allow: /*.svg
Allow: /*.ico

# Social media crawlers
User-agent: Twitterbot
Allow: /files/images/
Allow: /files/logos/

User-agent: facebookexternalhit
Allow: /files/images/
Allow: /files/logos/