# ------------------------------------------------
# Global rules for all crawlers
# ------------------------------------------------
User-agent: *
Allow: /

# Block sensitive or non-SEO friendly areas
Disallow: /admin/
Disallow: /login/
Disallow: /private/
Disallow: /api/
Disallow: /search
Disallow: /cgi-bin/

# WordPress-specific
Disallow: /wp-admin/
Disallow: /wp-login.php
Allow: /wp-admin/admin-ajax.php

# Prevent duplicate URLs with parameters
Disallow: /*?
Disallow: /*&

# Block internal search results (SEO best practice)
Disallow: /?s=

# ------------------------------------------------
# AI Crawlers — Allowed
# ------------------------------------------------

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Google AI (Gemini training opt-in)
User-agent: Google-Extended
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# xAI (Grok)
User-agent: xai-crawler
Allow: /

# Amazon AI
User-agent: Amazonbot
Allow: /

# Apple AI
User-agent: Applebot-Extended
Allow: /

# Common AI aggregators / newer bots
User-agent: Bytespider
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# ------------------------------------------------
# Crawl optimization (optional but recommended)
# ------------------------------------------------

# Reduce crawl load from aggressive bots
User-agent: *
Crawl-delay: 5

# ------------------------------------------------
# Sitemap
# ------------------------------------------------
Sitemap: https://www.ciklum.com/sitemap_index.xml