<?php
    $host_whitelist[]="\.ahrefs\.net$"; //ahrefs 
    $host_whitelist[]="\.crawl\.amazonbot\.amazon$"; //amazonbot
    $host_whitelist[]="\.crawl\.commoncrawl\.org$"; //commoncrawl
    $host_whitelist[]="\.fbsv\.net$"; //facebook
    $host_whitelist[]="\.googlebot\.com$"; //googlebot
    $host_whitelist[]="\.kaspersky-labs\.com$"; //kaspersky labs
    $host_whitelist[]="mj12bot\.com$"; //MJ12bot
    $host_whitelist[]="\.search\.msn\.com$"; //msnbot
    $host_whitelist[]="\.semrush\.com$"; //semrush
?>