# ------------------------------------------------------
# Google Search / Images / Shopping / Ads / Search Console
# ------------------------------------------------------

User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Video
User-agent: Googlebot-News
User-agent: Storebot-Google
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: Google-InspectionTool
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
Allow: /



# ------------------------------------------------------
# Bing / Microsoft
# ------------------------------------------------------

User-agent: Bingbot
User-agent: msnbot
User-agent: adidxbot
Allow: /
Crawl-delay: 5



# ------------------------------------------------------
# Meta / Facebook / Instagram / Messenger / WhatsApp preview
# ------------------------------------------------------

User-agent: facebookexternalhit
User-agent: Facebot
User-agent: FacebookBot
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: meta-externalads
User-agent: WhatsApp
Allow: /
Crawl-delay: 10



# ------------------------------------------------------
# Pinterest
# ------------------------------------------------------

User-agent: Pinterestbot
Allow: /
Crawl-delay: 10
Disallow: /cart
Disallow: /cart/
Disallow: /checkout
Disallow: /checkout/
Disallow: /account
Disallow: /account/
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /admin
Disallow: /admin/
Disallow: /api/
Disallow: /ajax/
Disallow: /order
Disallow: /orders
Disallow: /payment
Disallow: /payment/


# ------------------------------------------------------
# Optional useful bots
# ------------------------------------------------------

User-agent: DuckDuckBot
User-agent: Applebot
User-agent: Slurp
Allow: /
Crawl-delay: 10



# ------------------------------------------------------
# SEO tools / aggressive crawlers / scrapers - block
# ------------------------------------------------------

User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: SemrushBot-SA
User-agent: SemrushBot-SI
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: PetalBot
User-agent: Bytespider
User-agent: YandexBot
User-agent: Baiduspider
User-agent: Sogou
User-agent: Exabot
User-agent: SeznamBot
User-agent: CCBot
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Amazonbot
User-agent: omgilibot
User-agent: ImagesiftBot
User-agent: Scrapy
User-agent: HTTrack
User-agent: wget
User-agent: curl
Disallow: /


# ------------------------------------------------------
# Default: all other bots blocked
# ------------------------------------------------------

User-agent: *
Disallow: /