# Robots & Agent policy
# Checkouts are for humans.
# * Automated scraping, "buy-for-me" agents, or any end-to-end flow
#   that completes payment without a final human review step is not permitted.

User-agent: OAI-SearchBot
Allow: /
Disallow: /cart
Disallow: /carts
Disallow: /checkout
Disallow: /checkouts/
Disallow: /orders
Disallow: /search
Disallow: /api/*
Disallow: /categories/*?*sort=*
Disallow: /categories/*?*order=*
Disallow: /categories/*?*min_price=*
Disallow: /categories/*?*max_price=*
Disallow: /categories/*?*sort=*&*
Disallow: /categories/*+*
Disallow: /categories/*%2B*
Disallow: /categories/*%2b*

User-agent: *
Allow: /
Allow: /api/feeds/
Disallow: /cart
Disallow: /carts
Disallow: /checkout
Disallow: /checkouts/
Disallow: /orders
Disallow: /search
Disallow: /api/*
Disallow: /categories/*?*sort=*
Disallow: /categories/*?*order=*
Disallow: /categories/*?*min_price=*
Disallow: /categories/*?*max_price=*
Disallow: /categories/*?*sort=*&*
Disallow: /categories/*+*
Disallow: /categories/*%2B*
Disallow: /categories/*%2b*

# Google Ads bot — ignores wildcard rules unless explicitly named
User-agent: adsbot-google
Allow: /
Allow: /api/feeds/
Disallow: /checkout
Disallow: /checkouts/
Disallow: /cart
Disallow: /carts
Disallow: /orders

User-agent: Storebot-Google
Allow: /
Allow: /api/feeds/

# Block Nutch entirely (aggressive scraper)
User-agent: Nutch
Disallow: /

# Ahrefs bots — crawl-delay to reduce server load
User-agent: AhrefsBot
Crawl-delay: 10
Disallow: /cart
Disallow: /carts
Disallow: /checkout
Disallow: /checkouts/
Disallow: /orders
Disallow: /search
Disallow: /api/*
Disallow: /categories/*?*sort=*
Disallow: /categories/*?*order=*
Disallow: /categories/*?*min_price=*
Disallow: /categories/*?*max_price=*
Disallow: /categories/*?*sort=*&*
Disallow: /categories/*+*
Disallow: /categories/*%2B*
Disallow: /categories/*%2b*

User-agent: AhrefsSiteAudit
Crawl-delay: 10
Disallow: /cart
Disallow: /carts
Disallow: /checkout
Disallow: /checkouts/
Disallow: /orders
Disallow: /search
Disallow: /api/*
Disallow: /categories/*?*sort=*
Disallow: /categories/*?*order=*
Disallow: /categories/*?*min_price=*
Disallow: /categories/*?*max_price=*
Disallow: /categories/*?*sort=*&*
Disallow: /categories/*+*
Disallow: /categories/*%2B*
Disallow: /categories/*%2b*

# MJ12bot — slow down crawl
User-agent: MJ12bot
Crawl-delay: 10

# Pinterest — gentle crawl
User-agent: Pinterest
Crawl-delay: 1

Sitemap: https://kathabyanupama.com/sitemap.xml