User-agent: * Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /93417308473/checkouts Disallow: /93417308473/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote # Scuba Evolution: catalogue data endpoints Disallow: /products.json Disallow: /*/products.json Disallow: /collections/*/products.json Disallow: /*/collections/*/products.json Disallow: /products/*.json Disallow: /*/products/*.json Disallow: /products/*.js Disallow: /*/products/*.js Disallow: /collections/*.atom Disallow: /*/collections/*.atom Disallow: /search/suggest.json Disallow: /*/search/suggest.json Sitemap: https://scubaevolution.co.za/sitemap.xml User-agent: adsbot-google Disallow: /checkouts/ Disallow: /checkout Disallow: /carts Disallow: /orders Disallow: /93417308473/checkouts Disallow: /93417308473/orders Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /cdn/wpm/*.js Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /services/login_with_shop User-agent: Nutch Disallow: / User-agent: AhrefsSiteAudit Crawl-delay: 10 Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /93417308473/checkouts Disallow: /93417308473/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Sitemap: https://scubaevolution.co.za/sitemap.xml User-agent: Pinterest Crawl-delay: 1 # ============================================================================= # Scuba Evolution custom crawler rules # ============================================================================= # # ALLOWED ON PURPOSE - do not add these to the blocks below. # These are the engines that cite sources and send real traffic. Blocking them # would undo the answer-engine work done on the blog and course pages. # Googlebot, Bingbot, DuckDuckBot, Applebot, PetalBot (search) # OAI-SearchBot, ChatGPT-User (ChatGPT search + fetch) # Claude-User, Claude-SearchBot (Claude search + fetch) # PerplexityBot, Perplexity-User (Perplexity) # Google-Extended (Gemini grounding) # # Our own site-audit crawlers are allowed too: # AhrefsSiteAudit keeps Shopify's default group, untouched by the loop above. # It is a separate agent from AhrefsBot and is NOT caught by # the AhrefsBot block below. # SemrushBot-SA needs the explicit group below, because "SemrushBot" is a # prefix of it and would otherwise capture it. # # ----------------------------------------------------------------------------- # Our own audits: longest matching group wins, so these override the blocks. # ----------------------------------------------------------------------------- User-agent: SemrushBot-SA Allow: / User-agent: SiteAuditBot Allow: / # ----------------------------------------------------------------------------- # Block 1: competitor-facing SEO and market-intelligence indexers. # These crawl the catalogue and resell it as competitor research. Blocking them # also removes this site from our own view inside those tools. # AhrefsBot and MJ12bot have their Shopify default groups suppressed above, so # each appears exactly once, here. # ----------------------------------------------------------------------------- User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: rogerbot Disallow: / User-agent: BLEXBot Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: Barkrowler Disallow: / User-agent: serpstatbot Disallow: / User-agent: MegaIndex.ru Disallow: / User-agent: linkdexbot Disallow: / User-agent: SeekportBot Disallow: / User-agent: Screaming Frog SEO Spider Disallow: / # ----------------------------------------------------------------------------- # Block 2: training-only AI crawlers. # These take the copy and return no citation and no traffic. The retrieval bots # from the same companies are listed as allowed above and are NOT affected. # ----------------------------------------------------------------------------- User-agent: GPTBot Disallow: / User-agent: ClaudeBot Disallow: / User-agent: anthropic-ai Disallow: / User-agent: CCBot Disallow: / User-agent: Applebot-Extended Disallow: / User-agent: Bytespider Disallow: / User-agent: meta-externalagent Disallow: / User-agent: FacebookBot Disallow: / User-agent: Amazonbot Disallow: / User-agent: Diffbot Disallow: / User-agent: Omgilibot Disallow: / User-agent: omgili Disallow: / User-agent: ImagesiftBot Disallow: / User-agent: cohere-ai Disallow: / User-agent: Timpibot Disallow: / User-agent: Webzio-Extended Disallow: / User-agent: PanguBot Disallow: / User-agent: AI2Bot Disallow: / User-agent: Scrapy Disallow: /