# ==========================================
# robots.txt — Travel With Raeef
# Last updated: 2026-04-10
# ==========================================

# ── SEO crawler rate limits ────────────────
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: SemrushBot-SA
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 10

User-agent: MozBot
Crawl-delay: 10

User-agent: BLEXBot
Crawl-delay: 10

User-agent: DataForSeoBot
Crawl-delay: 10

# ── Default rules for all other bots ──────
User-agent: *

# ── Explicitly allow affiliate partner review pages (/out/) ──
Allow: /out/agoda
Allow: /out/booking
Allow: /out/esim
Allow: /out/getyourguide
Allow: /out/holafly
Allow: /out/insurance
Allow: /out/transport
Allow: /out/worldnomads
Allow: /out/

# ── Private / Admin paths ──────────────────
Disallow: /admin
Disallow: /admin/
Disallow: /sitemap-generator
Disallow: /sitemap-generator/
Disallow: /sitemap-visa-routes
Disallow: /sitemap-visa-routes/
Disallow: /subdomain
Disallow: /subdomain/
Disallow: /affiliate-redirect
Disallow: /affiliate-redirect/

# ── Public blog / article / post paths ────
Allow: /article/
Allow: /articles
Allow: /articles/

# ── Public tools & pages ──────────────────
Allow: /visa-checker
Allow: /visa-checker/
Allow: /visa-approval-probability-calculator
Allow: /passport-index
Allow: /passport-index/
Allow: /passport-comparison
Allow: /visa-comparison
Allow: /visa-documents
Allow: /document-templates
Allow: /document-info/
Allow: /countries
Allow: /faq
Allow: /about
Allow: /affiliate-hub
Allow: /affiliate-disclosure
Allow: /privacy-policy
Allow: /terms-of-service
Allow: /sitemap
Allow: /visa/

# ── Sitemap references ─────────────────────
Sitemap: https://raeef.com/sitemap.xml
Sitemap: https://raeef.com/sitemap-pages.xml
Sitemap: https://raeef.com/sitemap-blog.xml
Sitemap: https://raeef.com/sitemap-visa-routes-1.xml
Sitemap: https://raeef.com/sitemap-visa-routes-2.xml
Sitemap: https://raeef.com/sitemap-visa-routes-3.xml