# Robots.txt for QuillCircuit - Comprehensive Blogging Platform
User-agent: *
Allow: /
Crawl-delay: 10

# CRITICAL: Allow Google AdSense crawlers for ads.txt verification
User-agent: Mediapartners-Google
Allow: /
Crawl-delay: 1

User-agent: Google-Display-Ads-Bot
Allow: /
Crawl-delay: 1

User-agent: AdsBot-Google
Allow: /
Crawl-delay: 1

# Block specific directories or sensitive areas
Disallow: /admin/
Disallow: /private/
Disallow: /dashboard/
Disallow: /user/login
Disallow: /user/register
Disallow: /api/auth/

# Sitemaps
Sitemap: https://www.quillcircuit.com/sitemap.xml

# Specific crawler instructions for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 5

User-agent: Bingbot
Allow: /
Crawl-delay: 5

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 5

User-agent: facebookexternalhit
Allow: /
Crawl-delay: 5

User-agent: Twitterbot
Allow: /
Crawl-delay: 5

# Block aggressive bots and scrapers
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

# SEO Optimization Areas - Explicitly allow important sections
Allow: /blog/
Allow: /about/
Allow: /contact/
Allow: /categories/
Allow: /tags/

# Special handling for media directories
User-agent: Googlebot-Image
Allow: /images/
Allow: /media/
Allow: /assets/

User-agent: Googlebot-Video
Allow: /videos/
Allow: /media/