# ==== Block known SEO scrapers / non-search crawlers ====
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: SEOkicks
Disallow: /

User-agent: ltx71
Disallow: /

User-agent: XoviBot
Disallow: /

User-agent: SeobilityBot
Disallow: /

User-agent: spbot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SerpstatBot
Disallow: /

User-agent: LinkdexBot
Disallow: /

User-agent: Wotbox
Disallow: /

User-agent: CocolyzeBot
Disallow: /

User-agent: NetpeakSpider
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /

# ==== Block AI / data scraping bots ====
User-agent: GPTBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# ==== Catch-all for generic crawlers ====
User-agent: Crawler
Disallow: /

User-agent: python-requests
Disallow: /

User-agent: curl
Disallow: /

User-agent: wget
Disallow: /

User-agent: Java
Disallow: /

User-agent: Go-http-client
Disallow: /

User-agent: okhttp
Disallow: /

User-agent: scrapy
Disallow: /

# ==== Default rules for everyone else (Googlebot, Bingbot, etc.) ====
User-agent: *
# Block crawling of internal scripts, APIs, embeds, and cdn-cgi endpoints
Disallow: /UI/
Disallow: /cdn-cgi/
Disallow: /api/
Disallow: /adminp/

# Allow important public content
Allow: /movies/
Allow: /tv-shows/
Allow: /discover/
allow: /proxy.php
allow: /proxy2.php
Allow: /img/
Allow: /assets/
Allow: /sitemaps/
Allow: /favicon.ico
Allow: /

# ==== Sitemap ====
Sitemap: https://123moviies.net/sitemaps/sitemap-index.xml