User-agent: *
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /private/
Allow: /wp-admin/admin-ajax.php
Allow: /public/
Allow: /assets/
Allow: /images/
Allow: /videos/
Allow: /js/
Allow: /css/

# Kecepatan crawl agar efisien
Crawl-delay: 5

# Sitemap utama
Sitemap: https://rtpmantap89.co/sitemap.xml

# ================================
# Bot whitelist (search engine utama)
# ================================
User-agent: Googlebot
Disallow:
Crawl-delay: 2

User-agent: Bingbot
Disallow:
Crawl-delay: 3

User-agent: DuckDuckBot
Disallow:
Crawl-delay: 3

User-agent: Yandex
Disallow:
Crawl-delay: 3

User-agent: Baiduspider
Disallow:
Crawl-delay: 5

# ================================
# Bot blacklist (pengamanan dasar)
# ================================
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SEMrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: spbot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: YisouSpider
Disallow: /

User-agent: PetalBot
Disallow: /

# ================================
# Akhiran
# ================================
# File ini membantu Google, Bing, dan mesin pencari lain
# menelusuri konten dengan cepat tanpa menabrak direktori sensitif.
# Pastikan sitemap.xml selalu up-to-date.