# CellCog Robots.txt
# https://cellcog.ai
#
# Policy (Nitish via Rhea + Arjun, 2026-09-02, CEL-2097 D1): every public page is open to
# search crawlers, AI search crawlers, user-triggered AI fetchers AND model-training crawlers.
# Agents should start at https://cellcog.ai/llms.txt (index) and https://cellcog.ai/for-agents.
# Authenticated app routes, the private API, and agent-built apps stay disallowed for everyone.

User-agent: *
Allow: /

# Sitemap location
Sitemap: https://cellcog.ai/sitemap.xml
Sitemap: https://cellcog.ai/video-sitemap.xml

# Block admin and API routes from indexing (the public OpenAPI spec lives at /openapi.json, not under /api/)
Disallow: /admin/
Disallow: /api/

# Block authenticated-only pages from crawling
# These pages require login and won't be useful in search results
Disallow: /chat/
Disallow: /projects/
Disallow: /drive/
Disallow: /avatars/
Disallow: /organization/
Disallow: /profile/
Disallow: /scheduled/

# Block private agent-built apps (auth-gated; leak into crawl queues via shared links)
Disallow: /app/

# Allow public pages
Allow: /blog
Allow: /blog/
Allow: /ai-employees
Allow: /compare
Allow: /compare/
Allow: /benchmarks
Allow: /ai-organization
Allow: /super-agents
Allow: /agent-to-agent
Allow: /pricing
Allow: /contact
Allow: /linear
Allow: /openclaw
Allow: /hermes
Allow: /cursor
Allow: /claude-code
Allow: /codex
Allow: /posts
Allow: /posts/
Allow: /skills
Allow: /skills/
Allow: /support
Allow: /support/
Allow: /tutorials
Allow: /tutorials/
Allow: /podcast
Allow: /podcast/
Allow: /login
Allow: /signup
Allow: /invitation/

# Agent-facing surfaces (agent-readable web program, 2026-09)
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /openapi.json
Allow: /for-agents
Allow: /.well-known/

# Block query parameters that create duplicate content
Disallow: /*?tab=
Disallow: /*?redirect=

# ---------------------------------------------------------------------------
# AI crawlers, named by purpose. Same policy for all of them: full access.
# Tokens per each vendor's crawler docs (OpenAI, Anthropic, Perplexity, Google), Sep 2026.
# ---------------------------------------------------------------------------

# OpenAI: search index, user-triggered fetches, model training
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

# Anthropic: search index, user-triggered fetches, model training
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: ClaudeBot
Allow: /

# Perplexity: search index, user-triggered fetches
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google: Google-Extended is a policy token (Gemini training/grounding), not a separate crawler
User-agent: Google-Extended
Allow: /

# Search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /