User-agent: *
Allow: /
Disallow: /wp-admin/ 
Allow: /wp-admin/admin-ajax.php
Disallow: /category/
Disallow: /*?page_id=
Sitemap: https://itpathsolutions.com/sitemap_index.xml
 
 
# OpenAI (ChatGPT / GPTBot)
User-agent: GPTBot
Allow: /
 
# OpenAI's WebCrawler (used for ChatGPT Search / oai)
User-agent: ChatGPT-User
Allow: /
 
# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /
 
# Google (Gemini / Bard / AI Overviews)
User-agent: Google-Extended
Allow: /
 
# Perplexity.ai
User-agent: PerplexityBot
Allow: /
 
# Scale AI (data labeling)
User-agent: CCBot
Disallow: /
 
# Amazon (Titan / Alexa Teacher Model)
User-agent: Amazonbot
Allow: /
 
# AppleBot (Siri / Apple LLM training)
User-agent: Applebot-Extended
Allow: /
 
# Common scrapers to block
User-agent: DataForSeoBot
Disallow: /
 
User-agent: Bytespider
Disallow: /