# ============================================================
# AI CRAWLER RULES — All allowed for GEO visibility
# ============================================================

# OpenAI / ChatGPT — Training + live search both allowed
User-agent: GPTBot
Allow: /

# ChatGPT live web search
User-agent: ChatGPT-User
Allow: /

# Anthropic / Claude — Training + live search both allowed
User-agent: anthropic-ai
Allow: /

# Claude live web search
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

# Google Gemini — CRITICAL: was blocking 299 pages, now allowed
User-agent: Google-Extended
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# DeepSeek AI
User-agent: DeepSeekBot
Allow: /

# You.com Search AI
User-agent: YouBot
Allow: /

# Apple Generative AI
User-agent: Applebot-Extended
Allow: /

# Mistral AI
User-agent: MistralBot
Allow: /

# Common Crawl (used by many AI training datasets)
User-agent: CCBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# ============================================================
# BLOCK IRRELEVANT / SPAM BOTS
# ============================================================
User-agent: Baiduspider
Disallow: /

User-agent: 360spider
Disallow: /

User-agent: Sosospider
Disallow: /

User-agent: sogou spider
Disallow: /

# ============================================================
# GENERAL CRAWL RULES
# ============================================================
User-agent: *

# WordPress admin only — CSS/JS must remain crawlable for Google rendering
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /recommended/
Disallow: /comments/feed/
Disallow: /trackback/
Disallow: /index.php
Disallow: /xmlrpc.php

# Allow uploads folder so AI can index product images
Allow: /wp-content/uploads/

# Always allow llms.txt for AI engines
Allow: /llms.txt

# Reduced crawl delay (was 30, now 2 — good)
Crawl-delay: 2

# ============================================================
# SITEMAPS
# ============================================================
Sitemap: https://shipsage.com/sitemap_index.xml