# robots.txt for Laptop Part Finder
User-agent: *
Allow: /

# Static category hubs (MM-025 / CX-038) — crawlable typed /parts/ children
Allow: /parts/batteries/
Allow: /parts/screens/
Allow: /parts/keyboards/
Allow: /parts/power-adapters/
Allow: /parts/fans/

# Exclude internal search query parameters from crawling if necessary
Disallow: /search/?*
Disallow: /go/

# Bare /compare/ stays indexable. /compare/?* is crawlable so Google sees its noindex, follow (SEO wk1, 2026-10-30).
Allow: /compare/

# AI agents: see /llms.txt, the JSON API at /api/ and the MCP server at /mcp
Allow: /api/
Allow: /llms.txt
Allow: /openapi.json
Allow: /.well-known/

# AI search and agent crawlers: welcome. Data is free to use with citation.
# (A named group replaces the * group for these bots, so the * rules are repeated here.)
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: CCBot
User-agent: meta-externalagent
User-agent: DuckAssistBot
User-agent: MistralAI-User
Allow: /
Disallow: /search/?*
Disallow: /go/
Allow: /compare/

# SEO tool crawlers: slow down (Cloudflare WAF also challenges these on HTML pages).
# (A named group replaces the * group, so the * rules are repeated in each group.)
User-agent: SemrushBot
Allow: /
Disallow: /search/?*
Disallow: /go/
Allow: /compare/
Crawl-delay: 10

User-agent: AhrefsBot
Allow: /
Disallow: /search/?*
Disallow: /go/
Allow: /compare/
Crawl-delay: 10

User-agent: KeenableBot
Allow: /
Disallow: /search/?*
Disallow: /go/
Allow: /compare/
Crawl-delay: 10

User-agent: MJ12bot
Allow: /
Disallow: /search/?*
Disallow: /go/
Allow: /compare/
Crawl-delay: 10

User-agent: DotBot
Allow: /
Disallow: /search/?*
Disallow: /go/
Allow: /compare/
Crawl-delay: 10

User-agent: DataForSeoBot
Allow: /
Disallow: /search/?*
Disallow: /go/
Allow: /compare/
Crawl-delay: 10

User-agent: PetalBot
Allow: /
Disallow: /search/?*
Disallow: /go/
Allow: /compare/
Crawl-delay: 10

# XML Sitemap (index of sitemap-parts.xml, sitemap-models.xml, sitemap-guides.xml, sitemap-pages.xml)
Sitemap: https://laptoppartfinder.com/sitemap.xml