# ==============================================================
# robots.txt for promenta.com
# Last updated: June 2026
# Strategy: Full crawl access for all bots (SEO + AI/LLM)
# with targeted exclusions for admin, staging, and duplicate paths
# ==============================================================


# ---------------------------------------------------------------
# SECTION 1: STANDARD SEARCH ENGINES
# ---------------------------------------------------------------

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /


# ---------------------------------------------------------------
# SECTION 2: AI TRAINING & LLM CRAWLERS (Allowed)
# Permitting these bots ensures promenta.com content surfaces
# in AI-generated answers (ChatGPT, Perplexity, Claude, Gemini, etc.)
# ---------------------------------------------------------------

# OpenAI (ChatGPT, GPT-4o, Browsing)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic (Claude)
User-agent: anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

# Google (Gemini / Bard training)
User-agent: Google-Extended
Allow: /

# Meta AI (Llama training)
User-agent: FacebookBot
Allow: /

# Apple Intelligence
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Common Crawl (powers many open LLMs)
User-agent: CCBot
Allow: /

# Diffbot (knowledge graph / AI enrichment)
User-agent: Diffbot
Allow: /

# Bytedance / Doubao
User-agent: Bytespider
Allow: /

# Amazon Alexa / Rufus
User-agent: Amazonbot
Allow: /

# Mistral / Le Chat
User-agent: MistralBot
Allow: /


# ---------------------------------------------------------------
# SECTION 3: GLOBAL RULES (ALL OTHER BOTS)
# ---------------------------------------------------------------

User-agent: *

# Allow all high-value content paths explicitly
Allow: /
Allow: /capabilities/
Allow: /solutions/
Allow: /accelerators/
Allow: /success-stories/
Allow: /blogs/
Allow: /technical-briefs/
Allow: /use-cases/
Allow: /about/
Allow: /tag/
Allow: /page/
Allow: /privacy-policy/
Allow: /terms-and-conditions/

# Block admin and system paths
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-json/
Disallow: /secure-login/

# Block search results and filter pages (prevent duplicate content)
Disallow: /?s=
Disallow: /search/
Disallow: /?p=


# Block utility and low-value pages
Disallow: /xmlrpc.php
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/


# Block staging / dev environments if on same domain
Disallow: /staging/
Disallow: /dev/
Disallow: /test/

# Crawl rate hint (optional — helps prevent server overload)
Crawl-delay: 2


# ---------------------------------------------------------------
# SECTION 4: SITEMAP REFERENCES
# ---------------------------------------------------------------

Sitemap: https://promenta.com/sitemap_index.xml