# =============================================================================
# robots.txt for VoooAI
# Last Updated: 2026-05-10
# Format: RFC 9309 compliant (no intra-group blank lines, no Allow:/ stub)
# =============================================================================

# =============================================================================
# Global Rules for All Search Engines
# =============================================================================
User-agent: *
Crawl-delay: 1
# API & Backend
Disallow: /api/
Disallow: /api/doc/
Disallow: /admin/
Disallow: /backend/
# Test & Debug
Disallow: /test-
Disallow: /debug-
Disallow: /tests/
Disallow: /playwright_test/
Disallow: /orb_debug_extension/
# Config & Logs
Disallow: /.env
Disallow: /logs/
Disallow: /backups/
Disallow: /migrations/
Disallow: /instance/
# User Content & Uploads
Disallow: /uploads/reference_images/guest/
# Auth & Payment Pages (not useful for search index)
Disallow: /payment
Disallow: /my-subscription
Disallow: /task-management
Disallow: /access-keys
Disallow: /callback/
Disallow: /login
Disallow: /register
# Private (login-required) pages - prevent soft 404 in GSC
Disallow: /notifications
Disallow: /workflows
Disallow: /workflow/
# Search & Query URLs
Disallow: /search
Disallow: /*?*q=
# Tracking parameters (UTM, referral) - prevent duplicate content
Disallow: /*?*utm_
Disallow: /*&utm_
Disallow: /*?*ref=
Disallow: /*&ref=
Disallow: /*?*gclid=
Disallow: /*?*fbclid=
# Duplicate content from test query parameters
Disallow: /*?*test=
Disallow: /*&test=
# Legacy ?lang= query parameter
# NOTE: 不再 Disallow，改为让爬虫访问后收到客户端 301 重定向
# （见 frontend/index.html 顶部 ?lang= 重定向脚本）→ /zh/ 规范 URL，
# 这样 Google/Ahrefs 会将旧外链的 UR/PageRank 权重合并到新路径。
# Disallow 会直接丢弃权重，反而伤害 SEO。

# =============================================================================
# Google (Googlebot) - explicit group (does not inherit from User-agent: *)
# =============================================================================
User-agent: Googlebot
Disallow: /api/
Disallow: /api/doc/
Disallow: /admin/
Disallow: /backend/
Disallow: /test-
Disallow: /debug-
Disallow: /tests/
Disallow: /playwright_test/
Disallow: /orb_debug_extension/
Disallow: /.env
Disallow: /logs/
Disallow: /backups/
Disallow: /migrations/
Disallow: /instance/
Disallow: /uploads/reference_images/guest/
Disallow: /payment
Disallow: /my-subscription
Disallow: /task-management
Disallow: /access-keys
Disallow: /callback/
Disallow: /login
Disallow: /register
Disallow: /notifications
Disallow: /workflows
Disallow: /workflow/
Disallow: /search
Disallow: /*?*q=
Disallow: /*?*utm_
Disallow: /*&utm_
Disallow: /*?*ref=
Disallow: /*&ref=
Disallow: /*?*gclid=
Disallow: /*?*fbclid=
Disallow: /*?*test=
Disallow: /*&test=
# Legacy ?lang= query parameter: 不 Disallow，让爬虫能 follow 客户端 301 重定向以合并权重

User-agent: Googlebot-Image
Disallow:

User-agent: Googlebot-Video
Disallow:

# =============================================================================
# Bing (Bingbot)
# =============================================================================
User-agent: Bingbot
Crawl-delay: 1
Disallow: /api/
Disallow: /api/doc/
Disallow: /admin/
Disallow: /backend/
Disallow: /test-
Disallow: /debug-
Disallow: /tests/
Disallow: /playwright_test/
Disallow: /orb_debug_extension/
Disallow: /.env
Disallow: /logs/
Disallow: /backups/
Disallow: /migrations/
Disallow: /instance/
Disallow: /uploads/reference_images/guest/
Disallow: /payment
Disallow: /my-subscription
Disallow: /task-management
Disallow: /access-keys
Disallow: /callback/
Disallow: /login
Disallow: /register
Disallow: /notifications
Disallow: /workflows
Disallow: /workflow/
Disallow: /search
Disallow: /*?*q=
Disallow: /*?*utm_
Disallow: /*&utm_
Disallow: /*?*ref=
Disallow: /*&ref=
Disallow: /*?*gclid=
Disallow: /*?*fbclid=
Disallow: /*?*test=
Disallow: /*&test=
# Legacy ?lang= query parameter: 不 Disallow，让爬虫能 follow 客户端 301 重定向以合并权重

# =============================================================================
# Baidu
# =============================================================================
User-agent: Baiduspider
Disallow: /api/
Disallow: /api/doc/
Disallow: /admin/
Disallow: /backend/
Disallow: /test-
Disallow: /debug-
Disallow: /tests/
Disallow: /playwright_test/
Disallow: /orb_debug_extension/
Disallow: /.env
Disallow: /logs/
Disallow: /backups/
Disallow: /migrations/
Disallow: /instance/
Disallow: /uploads/reference_images/guest/
Disallow: /payment
Disallow: /my-subscription
Disallow: /task-management
Disallow: /access-keys
Disallow: /callback/
Disallow: /search

User-agent: Baiduspider-image
Disallow:

# =============================================================================
# Sogou (搜狗)
# =============================================================================
User-agent: Sogou web spider
Disallow: /api/
Disallow: /api/doc/
Disallow: /admin/
Disallow: /backend/
Disallow: /.env
Disallow: /payment
Disallow: /my-subscription
Disallow: /callback/
Disallow: /search

# =============================================================================
# 360 Search
# =============================================================================
User-agent: 360Spider
Disallow: /api/
Disallow: /api/doc/
Disallow: /admin/
Disallow: /backend/
Disallow: /.env
Disallow: /payment
Disallow: /my-subscription
Disallow: /callback/
Disallow: /search

# =============================================================================
# Yandex
# =============================================================================
User-agent: Yandex
Disallow: /api/
Disallow: /api/doc/
Disallow: /admin/
Disallow: /backend/
Disallow: /.env
Disallow: /payment
Disallow: /my-subscription
Disallow: /callback/
Disallow: /search

# =============================================================================
# DuckDuckGo
# =============================================================================
User-agent: DuckDuckBot
Disallow: /api/
Disallow: /api/doc/
Disallow: /admin/
Disallow: /backend/
Disallow: /.env
Disallow: /payment
Disallow: /my-subscription
Disallow: /callback/
Disallow: /search

# =============================================================================
# AI & LLM Crawlers (Answer Engine Optimization)
# 显式声明允许访问（Disallow: 空值 = allow all），确保 AEO 暴露面被 AI 爬虫收入。
# 2026-05 补齐：ClaudeBot（取代 anthropic-ai）/ Google-Extended / Applebot-Extended /
#        Meta-ExternalAgent / OAI-SearchBot / Bytespider。
# =============================================================================
User-agent: GPTBot
Crawl-delay: 2
Disallow:

User-agent: OAI-SearchBot
Crawl-delay: 2
Disallow:

User-agent: ChatGPT-User
Disallow:

User-agent: CCBot
Disallow:

User-agent: ClaudeBot
Crawl-delay: 2
Disallow:

User-agent: Claude-Web
Disallow:

User-agent: anthropic-ai
Disallow:

User-agent: Google-Extended
Disallow:

User-agent: Applebot-Extended
Disallow:

User-agent: Meta-ExternalAgent
Disallow:

User-agent: FacebookBot
Disallow:

User-agent: Bytespider
Crawl-delay: 2
Disallow:

User-agent: PerplexityBot
Crawl-delay: 2
Disallow:

User-agent: omgili
Disallow:

# =============================================================================
# Social Media Crawlers (for link previews)
# =============================================================================
User-agent: facebookexternalhit
Disallow:

User-agent: Twitterbot
Disallow:

User-agent: LinkedInBot
Disallow:

# =============================================================================
# Sitemap
# =============================================================================
Sitemap: https://voooai.com/sitemap-index.xml

# For crawler-related inquiries: support@voooai.com