# Control Claudebot &  PerplexityBot - ALLOWLIST APPROACH (only permit specific paths)
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: ClaudeBot
User-agent: claude-code
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: Googlebot
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Bingbot
User-agent: AzureAI-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: meta-externalagent
User-agent: meta-webindexer
User-agent: Amazonbot
User-agent: Baiduspider
User-agent: AliyunSecBot
User-agent: Bytespider
User-agent: PetalBot
User-agent: DotBot
User-agent: dotbot
User-agent: AhrefsBot
User-agent: YandexBot
User-agent: Bingbot
User-agent: Applebot
User-agent: GenomeCrawlerd
User-agent: Buck
User-agent: IbouBot
User-agent: scalaj
User-agent: 360Spider
User-agent: Twingly Recon
Disallow: /
Disallow: /blog/archives/
Disallow: /category/
Disallow: /webportfolio/
Disallow: /README_FILES/
Disallow: /cgi-bin/
Disallow: /cleverhack/
Disallow: /josie/
Disallow: /mt/
Disallow: /wp-admin/
Disallow: /wp-images/
Disallow: /wp-includes/
Disallow: /webportfolio/
Disallow: /errata/
Disallow: /xarchive/
Disallow: /folderx/
Disallow: /coursenotes/
Disallow: /page/
Disallow: /keybase.txt
Disallow: /2002/
Disallow: /2003/
Disallow: /2004/
Disallow: /2005/
Disallow: /2006/
Disallow: /2007/
Disallow: /2008/
Disallow: /2009/
Disallow: /2010/
Disallow: /2011/
Disallow: /2012/
Disallow: /2013/
Disallow: /2014/
Disallow: /2015/
Disallow: /2016/
Disallow: /2017/
Disallow: /2018/
Disallow: /2019/
Disallow: /2020/
Disallow: /2021/
Disallow: /2022/
Disallow: /archive22/
Disallow: /archive21/
Disallow: /archive20/
Disallow: /archive19/
Disallow: /archive18/
Disallow: /archive17/
Disallow: /archive16/
Disallow: /archive15/
Disallow: /archive14/
Disallow: /archive13/
Disallow: /archive12/
Disallow: /archive11/
Disallow: /archive10/
Disallow: /archive09/
Disallow: /archive08/
Disallow: /archive07/
Disallow: /archive06/
Disallow: /archive05/
Disallow: /archive04/
Disallow: /archive02/
Disallow: /archive01/
Disallow: /archive00/
Allow: /2026
Allow: /feed.xml
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /agents.md
Allow: /frontier-ai-lab-tracker
Allow: /neolab-and-emerging-ai-lab-tracker
Allow: /frontier-ai-lab-news
Allow: /frontier-ai-interesting-news
Allow: /ai-coding-landscape
Allow: /the-urgency-of-open-source-ai
Allow: /2026-ai-startup-founder-resources
Allow: /global-ai-regulations-a-comprehensive-2025-roundup
Allow: /the-great-frontier-ai-lab-product-war-is-just-beginning
Allow: /ai-market-maps-curated-intelligence-from-leading-vcs-analysts-practitioners
Allow: /speaking
Allow: /about
Allow: /img/
Allow: /archive
Allow: /llms.md
Allow: /artificial-intelligence-in-h2-2025
Allow: /challenges-agentic-ai-for-business
Allow: /anthropic-claude-plays-pokemon
Allow: /developer-communications-gone-wrong
Allow: /deepseek-proves-why-society-needs-to-understand-llms
Allow: /if-you-have-agi-why-sell-it-through-an-api
Allow: /if-ai-development-stopped-today
Allow: /what-im-seeing-in-ai-gtm-fall-2024
Allow: /ai-glossary



# BLOCK BLOCK
User-agent: RootEvidence/1.0
Disallow: /

# Block TurnitinBot entirely
User-agent: TurnitinBot
Disallow: /

# Block l9tcpid/l9explore entirely
User-agent: l9tcpid/v1.1.0
User-agent: l9tcpid
User-agent: l9explore/1.2.2
User-agent: l9explore
Disallow: /



# Default rules for all other crawlers
User-agent: *
Disallow: /blog/archives/
Disallow: /category/
Disallow: /webportfolio/
Disallow: /README_FILES/
Disallow: /cgi-bin/
Disallow: /cleverhack/
Disallow: /josie/
Disallow: /mt/
Disallow: /wp-admin/
Disallow: /wp-images/
Disallow: /wp-includes/
Disallow: /errata/
Disallow: /xarchive/
Disallow: /folderx/
Disallow: /coursenotes/
Disallow: /page/
Disallow: /keybase.txt
Disallow: /2002/
Disallow: /2003/
Disallow: /2004/
Disallow: /2005/
Disallow: /2006/
Disallow: /2007/
Disallow: /2008/
Disallow: /2009/
Disallow: /2010/
Disallow: /2011/
Disallow: /2012/
Disallow: /2013/
Disallow: /2014/
Disallow: /2015/
Disallow: /2016/
Disallow: /2017/
Disallow: /2018/
Disallow: /2019/
Disallow: /2020/
Disallow: /2021/
Disallow: /2022/
Disallow: /archive22/
Disallow: /archive21/
Disallow: /archive20/
Disallow: /archive19/
Disallow: /archive18/
Disallow: /archive17/
Disallow: /archive16/
Disallow: /archive15/
Disallow: /archive14/
Disallow: /archive13/
Disallow: /archive12/
Disallow: /archive11/
Disallow: /archive10/
Disallow: /archive09/
Disallow: /archive08/
Disallow: /archive07/
Disallow: /archive06/
Disallow: /archive05/
Disallow: /archive04/
Disallow: /archive02/
Disallow: /archive01/
Disallow: /archive00/
Allow: /2026
Allow: /feed.xml
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /agents.md
Allow: /frontier-ai-lab-tracker
Allow: /frontier-ai-lab-news
Allow: /frontier-ai-interesting-news
Allow: /ai-coding-landscape
Allow: /the-urgency-of-open-source-ai
Allow: /2026-ai-startup-founder-resources
Allow: /global-ai-regulations-a-comprehensive-2025-roundup
Allow: /the-great-frontier-ai-lab-product-war-is-just-beginning
Allow: /ai-market-maps-curated-intelligence-from-leading-vcs-analysts-practitioners
Allow: /speaking
Allow: /about
Allow: /img/
Allow: /archive
Allow: /llms.md
Allow: /artificial-intelligence-in-h2-2025
Allow: /challenges-agentic-ai-for-business
Allow: /anthropic-claude-plays-pokemon
Allow: /developer-communications-gone-wrong
Allow: /deepseek-proves-why-society-needs-to-understand-llms
Allow: /if-you-have-agi-why-sell-it-through-an-api
Allow: /if-ai-development-stopped-today
Allow: /what-im-seeing-in-ai-gtm-fall-2024
Allow: /ai-glossary

# Sitemap declaration
Sitemap: https://cleverhack.com/sitemap.xml