# ============================
# Mirah Belle - Robots.txt
# Optimized for Magento 2, SEO & Server Protection
# ============================

########################################
# Global rules for all good crawlers
########################################
User-agent: *
# --- Allow essential pages ---
Allow: /$
Allow: /collections/
Allow: /categories/
Allow: /shop-by-concern.html
Allow: /about-us
Allow: /contact/
Allow: /media/
Allow: /pub/media/
Allow: /static/version
Allow: /skin/

# --- Block sensitive or duplicate Magento system URLs ---
Disallow: /admin/
Disallow: /checkout/
Disallow: /cart/
Disallow: /customer/
Disallow: /customer/account/
Disallow: /customer/address/
Disallow: /wishlist/
Disallow: /onestepcheckout/
Disallow: /search/
Disallow: /catalogsearch/
Disallow: /sendfriend/
Disallow: /review/
Disallow: /compare/
Disallow: /tag/
Disallow: /tmp/
Disallow: /errors/
Disallow: /downloader/
Disallow: /index.php/
Disallow: /dev/
Disallow: /var/
Disallow: /cgi-bin/

# --- Block harmful parameter crawling (server load protection) ---
Disallow: /*?dir=
Disallow: /*?limit=
Disallow: /*?mode=
Disallow: /*?p=
Disallow: /*?q=
Disallow: /*?cat=
Disallow: /*?rat=
Disallow: /*?price=
Disallow: /*?utm_source
Disallow: /*?utm_medium
Disallow: /*?utm_campaign
Disallow: /*?sort
Disallow: /*SID=
Disallow: /*&p=

# --- Block bots from shipping & checkout JS endpoints ---
Disallow: /static/*/Magento_Checkout/
Disallow: /rest/default/V1/guest-carts/
Disallow: /rest/*/V1/guest-carts/

# --- Disallow duplicate content & scripts ---
Disallow: /*add-to-cart*
Disallow: /*sendfriend*
Disallow: /*wishlist*
Disallow: /catalog/product_compare/
Disallow: /catalog/product/view/
Disallow: /*index.php/
Disallow: /customer-service/
Disallow: /ajax/

########################################
# AI / LLM CRAWLERS — STRICT LIMITED ACCESS
########################################
# Allowed: Homepage, Categories, Subcategories, Collections, About & Contact
# Block everything else by default

# OpenAI GPTBot
User-agent: GPTBot
Disallow: /
Allow: /$
Allow: /skin-and-hair-care-beauty-products-in-india
Allow: /shop-by-concern
Allow: /skincare-and-hair-care-ingredients
Allow: /about-us
Allow: /contact/

# ChatGPT-User
User-agent: ChatGPT-User
Disallow: /
Allow: /$
Allow: /skin-and-hair-care-beauty-products-in-india
Allow: /shop-by-concern
Allow: /skincare-and-hair-care-ingredients
Allow: /about-us
Allow: /contact/

# Google AI (Google-Extended)
User-agent: Google-Extended
Disallow: /
Allow: /$
Allow: /skin-and-hair-care-beauty-products-in-india
Allow: /shop-by-concern
Allow: /skincare-and-hair-care-ingredients
Allow: /about-us
Allow: /contact/

# Anthropic (Claude)
User-agent: ClaudeBot
Disallow: /
Allow: /$
Allow: /skin-and-hair-care-beauty-products-in-india
Allow: /shop-by-concern
Allow: /skincare-and-hair-care-ingredients
Allow: /about-us
Allow: /contact/

User-agent: anthropic-ai
Disallow: /
Allow: /$
Allow: /skin-and-hair-care-beauty-products-in-india
Allow: /shop-by-concern
Allow: /skincare-and-hair-care-ingredients
Allow: /about-us
Allow: /contact/

# Perplexity
User-agent: PerplexityBot
Disallow: /
Allow: /$
Allow: /skin-and-hair-care-beauty-products-in-india
Allow: /shop-by-concern
Allow: /skincare-and-hair-care-ingredients
Allow: /about-us
Allow: /contact/

# CommonCrawl (AI training dataset)
User-agent: CCBot
Disallow: /
Allow: /$
Allow: /skin-and-hair-care-beauty-products-in-india
Allow: /shop-by-concern
Allow: /skincare-and-hair-care-ingredients
Allow: /about-us
Allow: /contact/

########################################
# Google Ads & Media Crawlers (full allow)
########################################
User-agent: AdsBot-Google
Allow: /

User-agent: Googlebot-Image
Allow: /media/
Allow: /pub/media/
Disallow: /media/tmp/

User-agent: Googlebot-Mobile
Allow: /

########################################
# Sitemap Locations
########################################
Sitemap: https://www.mirahbelle.com/sitemap.xml
Sitemap: https://www.mirahbelle.com/pub/sitemap.xml

########################################
# Crawl Delay for aggressive bots
########################################
Crawl-delay: 10

# --- End of Robots.txt ---