From a8464009aaa1e6f4f3e1125345309e504a121e06 Mon Sep 17 00:00:00 2001 From: robert Date: Mon, 15 Dec 2003 00:04:29 +0000 Subject: [PATCH] add bot detection --- include/classes/useragent.php-class | 199 +++++++++++++++++++++++++++- testbed/ua_test.php | 1 + 2 files changed, 198 insertions(+), 2 deletions(-) diff --git a/include/classes/useragent.php-class b/include/classes/useragent.php-class index 670e7e2..b8d83d6 100755 --- a/include/classes/useragent.php-class +++ b/include/classes/useragent.php-class @@ -12,12 +12,16 @@ class userAgent { // returns the User Agent brand name // var $version // the User Agent version + // var $bot + // bool: true if this agent is a bot // // function getBrand() // returns the User Agent Brand Name // function getVersion() // returns the User Agent version // + // function isbot() + // returns true if User Agent seems to be a bot // function isns() // returns true if User Agent seems to be Netscape brand, false if not // function isns4() @@ -50,11 +54,58 @@ class userAgent { // Mozilla/4.0 (compatible; MSIE 5.12; Mac_PowerPC) OmniWeb/4.1.1-v424.6 // Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4b) Gecko/20030516 Mozilla Firebird/0.6 // Mozilla/5.0 (Windows; U; WinNT4.0; en-US; rv:1.5a) Gecko/20030728 Mozilla Firebird/0.6.1 - + // Python-urllib/1.15 + // *** search bots: *** + // Scooter/3.3 + // Spinne/2.0 med_AH + // Vagabondo/2.0 MT (webagent at wise-guys dot nl) + // TurnitinBot/1.5 ( ">http://www.turnitin.com/robot/crawlerinfo.html) + // FAST-WebCrawler/3.x Multimedia (mm dash crawler at fast dot no) + // Firefly/1.0 (compatible; Mozilla 4.0; MSIE 5.5) + // Googlebot/2.1 (+ ">http://www.googlebot.com/bot.html) + // Scrubby/2.2 ( ">http://www.scrubtheweb.com/) + // psbot/0.1 (+ ">http://www.picsearch.com/bot.html) + // NG/1.0 + // URL_Spider_Pro/3.0 ( ">http://www.innerprise.net/usp-spider.asp)" + // Pompos/1.3 ">http://dir.com/pompos.html + // Szukacz/1.5 (robot; www.szukacz.pl/jakdzialarobot.html; info@szukacz.pl) + // ASPseek/1.2.10 + // NPBot-1/2.0 + // NetResearchServer/2.7(loopimprovements.com/robot.html) + // dloader(NaverRobot)/1.0 + // Mozilla/4.0 compatible ZyBorg/1.0 Daily Refresh Beta-d03 (wn.zyborg@looksmart.net; + // Mozilla/2.0 (compatible; Ask Jeeves/Teoma) + // Mozilla/5.0 (Slurp/si; slurp@inktomi.com; ">http://www.inktomi.com/slurp.html) + // Mozilla/5.0 [en] (compatible; Gulper Web Bot 0.2.4 www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot) + // Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; Girafabot; girafabot at girafa dot com; + // Mozilla/4.0 (efp@gmx.net) + // Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98) + // PingALink Monitoring Services 1.0 (http://www.pingalink.com) + // IlTrovatore-Setaccio (+ ">http://www.iltrovatore.it) + // Mercator-2.0 + // appie 1.1 (www.walhello.com) + // larbin_2.6.2 (larbin2.6.2@unspecified.mail) + // OWR_Crawler 0.1 + // search.ch V1.4.2 (spiderman@search.ch; + // WebFilter Robot 1.0 + // Openfind data gatherer, Openbot/3.0+(robot-response@openfind.com.tw;+ + // LinkWalker + // Internet Explorer 5.5 + // BaiDuSpider + // Mozilla/4.0 (compatible; B-l-i-t-z-B-O-T) + // B l i t z B O T @ t r i c u s . n e t (Mozilla compatible) + // sitecheck.internetseer.com (For more info see: ">http://sitecheck.internetseer.com) + // http://www.almaden.ibm.com/cs/crawler   [c01] + // ia_archiver + // Nutch + // Mozilla + // HeinrichderMiragoRobot + // dumbBot var $uastring; var $brand; var $version; + var $bot = false; function userAgent($ua_string = "") { // *** constructor *** @@ -68,7 +119,7 @@ class userAgent { // get UA brand and version $this->brand = "Unknown"; $this->version = 0; - if (ereg("([0-9a-zA-Z\.]+)/([0-9a-zA-Z\.+]+)", $this->uastring, $regs)) { + if (ereg("([0-9a-zA-Z\.()-]+)/([0-9a-zA-Z\.+]+)", $this->uastring, $regs)) { $this->brand = $regs[1]; // this is a reasonable default :) $this->version = $regs[2]; // this is a reasonable default :) } @@ -120,6 +171,141 @@ class userAgent { $this->brand = "AppleWebKit"; $this->version = $regs[1]; } + elseif (ereg("ZyBorg/([0-9a-zA-Z\.+]+)", $this->uastring, $regs)) { + $this->brand = "ZyBorg"; + $this->version = $regs[1]; + $this->bot = true; + } + elseif (ereg("Ask Jeeves/([0-9a-zA-Z\.+]+)", $this->uastring, $regs)) { + $this->brand = "Ask Jeeves"; + $this->version = $regs[1]; + $this->bot = true; + } + elseif (ereg("Slurp/([0-9a-zA-Z\.+]+)", $this->uastring, $regs)) { + $this->brand = "Slurp"; + $this->version = $regs[1]; + $this->bot = true; + } + elseif (ereg("Openbot/([0-9a-zA-Z\.+]+)", $this->uastring, $regs)) { + $this->brand = "Openbot"; + $this->version = $regs[1]; + $this->bot = true; + } + elseif (ereg("Mercator-([0-9a-zA-Z\.+]+)", $this->uastring, $regs)) { + $this->brand = "Mercator"; + $this->version = $regs[1]; + $this->bot = true; + } + elseif (ereg("appie ([0-9a-zA-Z\.+]+)", $this->uastring, $regs)) { + $this->brand = "appie"; + $this->version = $regs[1]; + $this->bot = true; + } + elseif (ereg("larbin_([0-9a-zA-Z\.+]+)", $this->uastring, $regs)) { + $this->brand = "larbin"; + $this->version = $regs[1]; + $this->bot = true; + } + elseif (ereg("Gulper Web Bot ([0-9a-zA-Z\.+]+)", $this->uastring, $regs)) { + $this->brand = "Gulper Web Bot"; + $this->version = $regs[1]; + $this->bot = true; + } + elseif (ereg("HTTrack ([0-9a-zA-Z\.+]+)", $this->uastring, $regs)) { + $this->brand = "HTTrack"; + $this->version = $regs[1]; + $this->bot = true; + } + elseif (ereg("OWR_Crawler ([0-9a-zA-Z\.+]+)", $this->uastring, $regs)) { + $this->brand = "OWR_Crawler"; + $this->version = $regs[1]; + $this->bot = true; + } + elseif (ereg("search.ch ([0-9a-zA-Z\.+]+)", $this->uastring, $regs)) { + $this->brand = "search.ch"; + $this->version = $regs[1]; + $this->bot = true; + } + elseif (ereg("WebFilter Robot ([0-9a-zA-Z\.+]+)", $this->uastring, $regs)) { + $this->brand = "WebFilter Robot"; + $this->version = $regs[1]; + $this->bot = true; + } + elseif (ereg("^LinkWalker", $this->uastring)) { + $this->brand = "LinkWalker"; + $this->version = ""; + $this->bot = true; + } + elseif (ereg("^Internet Explorer 5.5", $this->uastring)) { + $this->brand = "Unknown bot (IE5.5)"; + $this->version = ""; + $this->bot = true; + } + elseif (ereg("^Mozilla", $this->uastring)) { + $this->brand = "Unknown bot (Mozilla)"; + $this->version = ""; + $this->bot = true; + } + elseif (ereg("^BaiDuSpider", $this->uastring)) { + $this->brand = "BaiDuSpider"; + $this->version = ""; + $this->bot = true; + } + elseif (ereg("^ia_archiver", $this->uastring)) { + $this->brand = "ia_archiver"; + $this->version = ""; + $this->bot = true; + } + elseif (ereg("^PingALink", $this->uastring)) { + $this->brand = "PingALink"; + $this->version = ""; + $this->bot = true; + } + elseif (ereg("^IlTrovatore-Setaccio", $this->uastring)) { + $this->brand = "IlTrovatore-Setaccio"; + $this->version = ""; + $this->bot = true; + } + elseif (ereg("^Nutch", $this->uastring)) { + $this->brand = "Nutch"; + $this->version = ""; + $this->bot = true; + } + elseif (ereg("^dumbBot", $this->uastring)) { + $this->brand = "dumbBot"; + $this->version = ""; + $this->bot = true; + } + elseif (ereg("http://www.almaden.ibm.com/cs/crawler", $this->uastring)) { + $this->brand = "almaden crawler"; + $this->version = ""; + $this->bot = true; + } + elseif (ereg("B-l-i-t-z-B-O-T", $this->uastring) || ereg("B l i t z B O T @ t r i c u s . n e t", $this->uastring)) { + $this->brand = "BlitzBOT"; + $this->version = ""; + $this->bot = true; + } + elseif (ereg("sitecheck.internetseer.com", $this->uastring)) { + $this->brand = "internetseer"; + $this->version = ""; + $this->bot = true; + } + elseif (ereg("Girafabot", $this->uastring)) { + $this->brand = "Girafabot"; + $this->version = ""; + $this->bot = true; + } + elseif (ereg("efp@gmx.net", $this->uastring)) { + $this->brand = "efp"; + $this->version = ""; + $this->bot = true; + } + elseif (ereg("HeinrichderMiragoRobot", $this->uastring)) { + $this->brand = "HeinrichderMiragoRobot"; + $this->version = ""; + $this->bot = true; + } elseif (ereg("MSIE ([0-9a-zA-Z\.+]+)", $this->uastring, $regs)) { $this->brand = "Microsoft Internet Explorer"; $this->version = $regs[1]; @@ -129,10 +315,19 @@ class userAgent { $this->version = $regs[1]; if (intval($this->version) == 4) { $this->brand .= " Communicator"; } } + + $botArray = array("Scooter","Spinne","Vagabondo","TurnitinBot","FAST-WebCrawler","Firefly","Googlebot", + "Scrubby","psbot","NG","URL_Spider_Pro","Pompos","Szukacz","ASPseek","NPBot-1", + "dloader(NaverRobot)","NetResearchServer","","","","","","",""); + + if (in_array($this->brand, $botArray)) { + $this->bot = true; + } } function getBrand() { return $this->brand; } function getVersion() { return $this->version; } + function isbot() { return $this->bot; } function isns() { // set it static so that we don't have to call it that often diff --git a/testbed/ua_test.php b/testbed/ua_test.php index 37f2f21..4fe45a7 100644 --- a/testbed/ua_test.php +++ b/testbed/ua_test.php @@ -23,6 +23,7 @@ print("
This is ".($ua->isns4()?"a":"no")." Netscape Communicator 4 br print("
This is ".($ua->isie()?"an":"no")." Internet Explorer browser.\n"); print("
This is ".($ua->geckobased()?"a":"no")." Gecko-based browser.\n"); print("
This is ".($ua->khtmlbased()?"a":"no")." KHTML-based browser.\n"); +print("
This is ".($ua->isbot()?"an":"no")." automated robot.\n"); print("
The browser brand is reported as "".$ua->getBrand().""\n"); print("
The browser version is reported as "".$ua->getVersion().""\n"); if ($ua->geckobased()) { print("
The Gecko date is reported as "".$ua->geckodate().""\n"); } -- 2.35.3