# Tử Vi AI - robots.txt
# Full SEO configuration for 2,400+ programmatic pages

User-agent: *
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /wallet
Disallow: /my-readings
Disallow: /change-password
Disallow: /*.json
Disallow: /*.xml

# Allow search engines to crawl sitemaps and specific paths
Allow: /sitemap.xml
Allow: /sitemap-*.xml

# AI crawlers - allow content mining with proper attribution
# OpenAI GPT family (ChatGPT, GPT-4, etc.)
User-agent: GPTBot
Allow: /

# Anthropic Claude API
User-agent: ClaudeBot
Allow: /

# Google Bard (Gemini)
User-agent: Google-Extended
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity
Allow: /

# Common Crawl (training data archive)
User-agent: CCBot
Allow: /

# ChatGPT API users (if enabled)
User-agent: ChatGPT-User
Allow: /

# Other AI/training bots
User-agent: OtherBot
Allow: /
User-agent: DataForSeoBot
Allow: /
User-agent: Apptiva
Allow: /

# Slow down aggressive crawlers
User-agent: AhrefsBot
Crawl-delay: 5

User-agent: SemrushBot
Crawl-delay: 5

# Sitemap tổng (index → các file con: main, blog, tuvi, …)
Sitemap: https://diencamtamthe.com/sitemap.xml