# MCPize - Model Context Protocol Server Marketplace # https://mcpize.com # Canonical: dynamic robots.txt is served from /src/routes/robots[.]txt.ts. # This static file is a fallback for direct fetches. User-agent: * Allow: / Disallow: /admin/ Disallow: /api/private/ Disallow: /*.json$ Disallow: /payment-success Disallow: /payment-cancel Disallow: /dashboard Disallow: /settings Disallow: /developer/ Disallow: /account/ Disallow: /auth # Search engines User-agent: Googlebot Allow: / Crawl-delay: 0 User-agent: Bingbot Allow: / Crawl-delay: 0 # Social previews User-agent: Twitterbot Allow: / User-agent: facebookexternalhit Allow: / User-agent: LinkedInBot Allow: / # AI crawlers - explicit allow with no throttle. # Each named so absence of an explicit block can't be read as ambiguity. User-agent: GPTBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 0 User-agent: OAI-SearchBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 0 User-agent: ChatGPT-User Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 0 User-agent: ClaudeBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 0 User-agent: Claude-Web Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 0 User-agent: Anthropic-AI Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 0 User-agent: PerplexityBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 0 User-agent: Perplexity-User Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 0 User-agent: Google-Extended Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 0 User-agent: Applebot-Extended Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 0 User-agent: CCBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 0 User-agent: Amazonbot Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 0 User-agent: Meta-ExternalAgent Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 0 User-agent: cohere-ai Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 0 User-agent: Bytespider Disallow: / Sitemap: https://mcpize.com/sitemap.xml