# =====================================================
# robots.txt 鈥" Balwaan Krishi (balwaan.com)
# E-commerce site: agricultural machinery & farm equipment
# Last updated: 2026-09-11
# =====================================================

# -----------------------------------------------------
# STANDARD SEARCH ENGINE CRAWLERS
# Allow full product/category/blog crawling.
# Block account, transaction, and internal-search paths.
# -----------------------------------------------------
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: YandexBot
Disallow: /*testing
Disallow: /*test-sandbox
Disallow: /*order
Disallow: /*admin
Disallow: /*change
Disallow: /*login
Disallow: /*cart
Disallow: /*payment
Disallow: /*checkout
Disallow: /*Checkout
Disallow: /*fav
Disallow: /*favourite
Disallow: /*profile
Disallow: /*password
Disallow: /*registration
Disallow: /*cancel
Disallow: /*order-successful
Disallow: /*library
Disallow: /*.html
Disallow: /*refer-and-earn
Disallow: /*ReferandEarn
Disallow: /*refund-policy
Disallow: /*validate_promo_code
Disallow: /*get_slider_images
Disallow: /*get_home_banner
Disallow: /*get_usages_tags
Disallow: /*get_categories
Disallow: /*get_sections
Disallow: /*get_banner_images
Disallow: /*getItemWiseCashback/
Disallow: /*Search/AutoComplete/
Disallow: /*google-login
Disallow: /*submitQuestion
Disallow: /*pincodeInfo
Disallow: /*questionAnswerList
Disallow: /head/
Disallow: /*style
Disallow: /*body
Allow: /global/spares


# -----------------------------------------------------
# GENERIC FALLBACK (any crawler not listed above)
# -----------------------------------------------------
User-agent: *
Disallow: /*testing
Disallow: /*test-sandbox
Disallow: /*order
Disallow: /*admin
Disallow: /*change
Disallow: /*login
Disallow: /*cart
Disallow: /*payment
Disallow: /*checkout
Disallow: /*Checkout
Disallow: /*fav
Disallow: /*favourite
Disallow: /*profile
Disallow: /*password
Disallow: /*registration
Disallow: /*cancel
Disallow: /*order-successful
Disallow: /*library
Disallow: /*.html
Disallow: /*refer-and-earn
Disallow: /*ReferandEarn
Disallow: /*refund-policy
Disallow: /*validate_promo_code
Disallow: /*get_slider_images
Disallow: /*get_home_banner
Disallow: /*get_usages_tags
Disallow: /*get_categories
Disallow: /*get_sections
Disallow: /*get_banner_images
Disallow: /*getItemWiseCashback/
Disallow: /*Search/AutoComplete/
Disallow: /*google-login
Disallow: /*submitQuestion
Disallow: /*pincodeInfo
Disallow: /*questionAnswerList
Disallow: /head/
Disallow: /*style
Disallow: /*body


# =====================================================
# LLM / AI CRAWLERS
# Allow indexing of public informational & product-listing
# content (helps AI Overviews / GEO visibility). Block
# transactional, account, and internal-API paths 鈥" same
# to protect server load from bulk AI scraping.
# =====================================================

# --- OpenAI ---
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
Disallow: /*order
Disallow: /*admin
Disallow: /*login
Disallow: /*cart
Disallow: /*payment
Disallow: /*checkout
Disallow: /*Checkout
Disallow: /*profile
Disallow: /*password
Disallow: /*registration
Disallow: /*fav
Disallow: /*favourite
Disallow: /*google-login
Disallow: /*Search/AutoComplete/
Disallow: /*get_*


# --- Google's AI training crawler (separate from Googlebot) ---
User-agent: Google-Extended
Disallow: /*order
Disallow: /*admin
Disallow: /*login
Disallow: /*cart
Disallow: /*payment
Disallow: /*checkout
Disallow: /*Checkout
Disallow: /*profile
Disallow: /*password
Disallow: /*registration


# --- Anthropic ---
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: ClaudeBot
Disallow: /*order
Disallow: /*admin
Disallow: /*login
Disallow: /*cart
Disallow: /*payment
Disallow: /*checkout
Disallow: /*Checkout
Disallow: /*profile
Disallow: /*password
Disallow: /*registration
Disallow: /*fav
Disallow: /*favourite


# --- Perplexity ---
User-agent: PerplexityBot
Disallow: /*order
Disallow: /*admin
Disallow: /*login
Disallow: /*cart
Disallow: /*payment
Disallow: /*checkout
Disallow: /*Checkout
Disallow: /*profile
Disallow: /*password
Disallow: /*registration


# --- Other major AI crawlers ---
User-agent: CCBot
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
User-agent: Bytespider
User-agent: Amazonbot
User-agent: Applebot-Extended
User-agent: cohere-ai
User-agent: Diffbot
User-agent: YouBot
Disallow: /*order
Disallow: /*admin
Disallow: /*login
Disallow: /*cart
Disallow: /*payment
Disallow: /*checkout
Disallow: /*Checkout
Disallow: /*profile
Disallow: /*password
Disallow: /*registration
Disallow: /*fav
Disallow: /*favourite


# =====================================================
# SITEMAP
# =====================================================
Sitemap: https://balwaan.com/sitemap.xml