# ------------------------------------------------------------
# ROBOTS.TXT FOR BEST TALENT REACH
# Updated: June 2026
# ------------------------------------------------------------

# Global rule: Allow all standard bots to crawl the public site
User-agent: *
Allow: /
Disallow: /app/

# ==========================================
# MAJOR SEARCH ENGINES
# ==========================================

User-agent: Googlebot
Disallow: /app/

User-agent: Googlebot-Image
Disallow: /app/

User-agent: Bingbot
Disallow: /app/

# Apple Search
User-agent: Applebot
Disallow: /app/

# ==========================================
# SOCIAL MEDIA CRAWLERS & SCRAPERS
# ==========================================

# Facebook Link Preview
User-agent: facebookexternalhit
Allow: /
Disallow: /app/

# Twitter / X Link Preview
User-agent: Twitterbot
Allow: /
Disallow: /app/

# LinkedIn Link Preview
User-agent: LinkedInBot
Allow: /
Disallow: /app/

# ==========================================
# AI SEARCH & RETRIEVAL AGENTS (Allow Open Search)
# ==========================================

# OpenAI ChatGPT (Real-time Search & User-initiated actions)
User-agent: OAI-SearchBot
Disallow: /app/

User-agent: ChatGPT-User
Disallow: /app/

# Anthropic Claude (Real-time Search & User-initiated actions)
User-agent: Claude-SearchBot
Disallow: /app/

User-agent: Claude-User
Disallow: /app/

# ==========================================
# SITEMAPS
# ==========================================
Sitemap: https://besttalentreach.com/sitemap.xml
Sitemap: https://besttalentreach.com/super-sitemap.xml