# ------------------------------------------------
# Global rules for all crawlers
# ------------------------------------------------
User-agent: *
Allow: /

# Block sensitive or non-SEO friendly areas
Disallow: /admin/
Disallow: /login/
Disallow: /private/
Disallow: /api/
Disallow: /search
Disallow: /cgi-bin/

# WordPress-specific
Disallow: /wp-admin/
Disallow: /wp-login/
Allow: /wp-admin/admin-ajax.php

# Block URL parameters (prevents duplicate indexing)
Disallow: /*?
Disallow: /*&

# ------------------------------------------------
# AI Crawlers — Fully Allowed
# ------------------------------------------------

# OpenAI – ChatGPT (GPTBot)
User-agent: GPTBot
Allow: /

# OpenAI – Search Crawler for o1 models
User-agent: OAI-SearchBot
Allow: /

# Google AI (Gemini training)
User-agent: Google-Extended
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Anthropic Claude AI
User-agent: ClaudeBot
Allow: /

# xAI – Grok bot
User-agent: xai-crawler
Allow: /

# ------------------------------------------------
# Sitemap
# ------------------------------------------------
Sitemap: https://www.ciklum.com/sitemap_index.xml