From d6547dc428dad5317dae5610ff4f3c7b653dbbc0 Mon Sep 17 00:00:00 2001 From: Marcel Werk Date: Tue, 20 Oct 2026 23:40:43 +0200 Subject: [PATCH] Prune outdated and overly generic spider identifiers --- .../event/spider/SpiderCollecting.class.php | 448 +----------------- 1 file changed, 8 insertions(+), 440 deletions(-) diff --git a/wcfsetup/install/files/lib/event/spider/SpiderCollecting.class.php b/wcfsetup/install/files/lib/event/spider/SpiderCollecting.class.php index 009ecaf9b8b..22aca87cd4d 100644 --- a/wcfsetup/install/files/lib/event/spider/SpiderCollecting.class.php +++ b/wcfsetup/install/files/lib/event/spider/SpiderCollecting.class.php @@ -22,111 +22,26 @@ final class SpiderCollecting implements IPsr14Event public function __construct() { - $this->register(new Spider('ABCdatos', 'ABCdatos BotLink', 'http://www.robotstxt.org/db/abcdatos.html')); - $this->register(new Spider('AbraveSpider', 'Abrave', 'http://robot.abrave.co.uk/')); - $this->register(new Spider('Accelatech RSSCrawler', 'Accelatech')); - $this->register(new Spider('Accoona-AI-Agent', 'Accoona', 'https://www.accoona.com/about/')); - $this->register(new Spider('aconon Index', 'aconon Index (raubfische.de)')); - $this->register(new Spider('AcoonBot', 'Acoon', 'http://www.acoon.de/robot.asp')); - $this->register(new Spider('Ahoy!', 'Ahoy!', 'http://www.robotstxt.org/db/ahoythehomepagefinder.html')); $this->register(new Spider('AhrefsBot', 'Ahrefs', 'http://ahrefs.com/robot/')); - $this->register(new Spider('AlkalineBOT', 'Alkaline', 'http://www.robotstxt.org/db/Alkaline.html')); $this->register(new Spider('Amazonbot', 'Amazonbot', 'https://developer.amazon.com/support/amazonbot')); - $this->register(new Spider('AlexaBOT', 'Alexa', 'http://www.alexa.com/help/webmasters')); - $this->register( - new Spider( - 'http://www.almaden.ibm.com/cs/crawler', - 'Almaden Crawler', - 'http://www.almaden.ibm.com/cs/crawler/' - ) - ); $this->register(new Spider('Barkrowler', 'Barkrowler', 'https://www.babbar.tech/crawler')); - $this->register(new Spider('EMC Spider', 'ananzi')); - $this->register(new Spider('Anthill', 'Anthill', 'http://www.robotstxt.org/db/anthill.html')); - $this->register(new Spider('Aport', 'Aport', 'http://www.aport.ru/')); - $this->register(new Spider('AppleSyndication', 'Apple')); $this->register(new Spider('Applebot', 'Applebot', 'https://support.apple.com/en-gb/HT204683')); - $this->register( - new Spider('Arachnophilia', 'Arachnophilia', 'http://www.robotstxt.org/db/arachnophilia.html') - ); - $this->register(new Spider('Araneo', 'Araneo', 'http://www.robotstxt.org/db/araneo.html')); - $this->register( - new Spider('ArchitextSpider', 'ArchitextSpider', 'http://www.robotstxt.org/db/architext.html') - ); $this->register(new Spider('archive.org_bot', 'Archive.org', 'https://archive.org/details/archive.org_bot')); - $this->register(new Spider('arks/1.0', 'arks', 'http://www.robotstxt.org/db/arks.html')); - $this->register(new Spider('ASpider', 'ASpider', 'http://www.robotstxt.org/db/aspider.html')); - $this->register(new Spider('ATN_Worldwide', 'ATN Worldwide', 'http://www.robotstxt.org/db/atn.html')); - $this->register(new Spider('Atomz', 'Atomz.com', 'http://www.robotstxt.org/db/atomz.html')); - $this->register(new Spider('AURESYS', 'AURESYS', 'http://www.robotstxt.org/db/auresys.html')); $this->register(new Spider('AwarioSmartBot', 'AwarioSmartBot', 'https://awario.com/bots.html')); $this->register(new Spider('AwarioRssBot', 'AwarioRssBot', 'https://awario.com/bots.html')); - $this->register(new Spider('BackRub', 'BackRub', 'http://www.robotstxt.org/db/backrub.html')); $this->register(new Spider('Baiduspider', 'Baiduspider', 'http://www.baidu.com/search/spider.htm')); - $this->register(new Spider('BecomeBot', 'BecomeBot', 'http://www.become.com/webmasters.html')); - $this->register(new Spider('Big Brother', 'Big Brother', 'http://www.robotstxt.org/db/bigbrother.html')); - $this->register(new Spider('BigmirSpider', 'Bigmir', 'http://www.bigmir.net/')); $this->register(new Spider('bingbot', 'Bing', 'http://www.bing.com/bingbot.htm')); - $this->register(new Spider('Bitacle bot', 'Bitacle', 'http://bitacle.org/')); $this->register(new Spider('BitBot', 'BitBot', 'https://bitbot.dev')); - $this->register(new Spider('Biz360 Spider', 'Biz', 'http://www.biz360.com')); - $this->register(new Spider('Bjaaland', 'Bjaaland', 'http://www.robotstxt.org/db/bjaaland.html')); - $this->register(new Spider('BlackWidow', 'BlackWidow', 'http://www.robotstxt.org/db/blackwidow.html')); $this->register(new Spider('BLEXBot', 'BLEXBot', 'http://webmeup.com/crawler.html')); - $this->register(new Spider('BlogCrawler by Xango', 'BlogCrawler')); - $this->register(new Spider('blogdb', 'BlogDb', 'http://blogdb.jp')); - $this->register(new Spider('blog search engine by BlogFan.ORG', 'BlogFan', 'http://www.blogfan.org')); - $this->register(new Spider('Bloglines', 'Bloglies', 'http://www.bloglines.com')); - $this->register(new Spider('BlogPulse (ISSpider-3.0)', 'BlogPulse')); - $this->register(new Spider('BlogSearch', 'BlogSearch', 'http://www.icerocket.com')); - $this->register(new Spider('BlogsNowBot', 'BlogsNow', 'http://www.blogsnow.com/')); - $this->register(new Spider('BlogStreetBot', 'BlogStreetBot', 'http://www.blogstreet.com/')); - $this->register(new Spider('Bulkfeeds', 'BlogStreetBot', 'http://bulkfeeds.net')); - $this->register(new Spider('BoardPulse', 'BoardPulse', 'http://www.boardpulse.com/')); $this->register(new Spider('BoardReader', 'BoardReader', 'http://www.boardreader.com/aboutus.asp')); - $this->register(new Spider('BoardViewer', 'BoardViewer', 'http://www.boardviewer.com/')); - $this->register(new Spider('boitho.com-robot', 'Boitho', 'http://www.boitho.com/bot.html')); - $this->register(new Spider('borg-bot', 'Borg-Bot', 'http://www.robotstxt.org/db/borg-bot.html')); - $this->register(new Spider('BSpider', 'BSpider', 'http://www.robotstxt.org/db/bspider.html')); $this->register(new Spider('BublupBot', 'BublupBot', 'https://www.bublup.com/bublup-bot.html')); $this->register(new Spider('Buck/2.2', 'Buck', 'https://app.hypefactors.com/media-monitoring/about.html')); - $this->register(new Spider('CACTVS Chemistry Spider', 'CACTVS Chemistry')); - $this->register(new Spider('Calif', 'Calif')); - $this->register(new Spider('CaRP/3.6Evolution', 'CaRP', 'http://www.biz360.com')); $this->register(new Spider('CensysInspect', 'Censys', 'https://about.censys.io/')); - $this->register(new Spider('Checkbot', 'Checkbot')); - $this->register(new Spider('ChristCrawler.com', 'ChristCrawler.com')); - $this->register(new Spider('www.cienciaficcion.net', 'cIeNcIaFiCcIoN.nEt')); - $this->register(new Spider('Cincraw', 'Cincraw', 'http://cincrawdata.net/bot/')); - $this->register(new Spider('CipinetBot', 'Cipinet', 'http://www.cipinet.com/bot.html')); - $this->register(new Spider('CJNetworkQuality', 'CJNetworkQuality', 'http://www.cj.com/networkquality/')); - $this->register(new Spider('CMC/0.01', 'CMC/0.01')); $this->register(new Spider('ColdFusion', 'ColdFusion')); - $this->register(new Spider('combine', 'Combine System')); - $this->register(new Spider('Crawler (cometsearch@cometsystems.com)', 'cometsystems.com')); - $this->register(new Spider('ComputingSite Robi/1.0', 'ComputingSite Robi/1.0')); - $this->register(new Spider('conceptbot', 'Conceptbot')); $this->register(new Spider('Cookiebot', 'Cookiebot', 'https://www.cookiebot.com/')); - $this->register(new Spider('Cooby.de Crawler', 'Cooby.de Crawler')); - $this->register(new Spider('CoolBot', 'CoolBot')); - $this->register(new Spider('Cusco', 'Cusco')); - $this->register(new Spider('CyberSpyder', 'CyberSpyder')); - $this->register(new Spider('daypopbot', 'daypop')); - $this->register(new Spider('DesertRealm.com', 'Desert Realm')); - $this->register(new Spider('Deweb', 'DeWeb(c)')); - $this->register(new Spider('Die Blinde Kuh', 'Die Blinde Kuh', 'http://www.robotstxt.org/db/blindekuh.html')); - $this->register(new Spider('dienstspider', 'DienstSpider')); - $this->register(new Spider('Digger/1.0 JDK/1.3.0', 'Digger')); - $this->register(new Spider('Digimarc WebReader', 'Digimarc MarcSpider')); - $this->register(new Spider('Digimarc CGIReader', 'Digimarc Marcspider/CGI')); - $this->register(new Spider('DIIbot', 'Digital Integrity Robot')); - $this->register(new Spider('grabber', 'Direct Hit Grabber')); - $this->register(new Spider('discobot', 'Discovery', 'http://discoveryengine.com/discobot.html')); $this->register(new Spider('Discordbot', 'Discord', 'https://discordapp.com/')); - $this->register(new Spider('DNAbot/1.0', 'DNAbot')); $this->register(new Spider('Domains Project', 'Domains Project', 'https://domainsproject.org/')); $this->register(new Spider('DotBot', 'Moz DotBot', 'http://www.opensiteexplorer.org/dotbot')); - $this->register(new Spider('DragonBot/1.0 libwww/5.0', 'DragonBot')); $this->register( new Spider( 'dragonmetrics', @@ -148,24 +63,7 @@ public function __construct() 'https://help.duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/' ) ); - $this->register(new Spider('DWCP/2.0', 'DWCP (Dridus\' Web Cataloging Project)')); - $this->register( - new Spider('e-SocietyRobot', 'e-Society', 'http://www.yama.info.waseda.ac.jp/~yamana/es/index_eng.htm') - ); - $this->register(new Spider('exactseek-pagereaper', 'eaxactseek-page')); - $this->register(new Spider('EbiNess/0.01a', 'EbiNess')); - $this->register(new Spider('edgeio-retriever', 'Edgeio', 'http://www.edgeio.com')); - $this->register(new Spider('EIT-Link-Verifier-Robot/0.2', 'EIT Link Verifier Robot')); - $this->register(new Spider('elfinbot', 'ELFINBOT')); - $this->register(new Spider('Emacs-w3/v[0-9\\.]+', 'Emacs-w3 Search Engine')); - $this->register(new Spider('esther', 'Esther')); - $this->register(new Spider('EuripBot/', 'EuripBot')); $this->register(new Spider('ev-crawler', 'Headline Web Crawler', 'https://headline.com/legal/crawler')); - $this->register(new Spider('Evliya Celebi', 'Evliya Celebi')); - $this->register(new Spider('ExactSeek_Spider', 'ExactSeek_Spider', 'http://www.askjeevs.com')); - $this->register(new Spider('NG/2.0', 'ExaLead', 'http://botspotter.net/bs-389.html')); - $this->register(new Spider('ExaBot', 'ExaLead Beta', 'http://beta.exalead.com/search/C=0/2p=Help.7')); - $this->register(new Spider('ExaLead', 'ExaLead', 'http://www.exalead.com/search')); $this->register( new Spider( 'facebookexternalhit', @@ -173,11 +71,6 @@ public function __construct() 'https://developers.facebook.com/docs/sharing/webmasters/crawler/' ) ); - $this->register( - new Spider('fast-webcrawler', 'FAST / AlltheWeb', 'http://help.yahoo.com/help/us/ysearch/slurp/index.html') - ); - $this->register(new Spider('FastCrawler', 'FastCrawler')); - $this->register(new Spider('Feed24.com', 'Feed24', 'http://www.feed24.com')); $this->register(new Spider('FeedBlitz', 'FeedBlitz', 'http://www.feedblitz.com')); $this->register(new Spider('FeedBurner', 'FeedBurner', 'http://www.FeedBurner.com')); $this->register(new Spider('Feedly', 'Feedly', 'http://www.feedly.com/fetcher.html')); @@ -198,19 +91,8 @@ public function __construct() 'https://support.google.com/webmasters/answer/9008080' ) ); - $this->register(new Spider('Google-Extended', 'Google-Extended')); $this->register(new Spider('UniversalFeedParser', 'FeedParser', 'http://www.feedparser.org')); - $this->register(new Spider('Feedster Crawler', 'Feedster', 'http://www.feedstermedia.com/')); - $this->register(new Spider('FEHLSTART Superspider', 'FEHLSTART')); - $this->register(new Spider('FelixIDE', 'Felix IDE')); - $this->register(new Spider('ESIRover', 'FetchRover')); - $this->register(new Spider('fido', 'fido')); $this->register(new Spider('findlinks', 'FindLinks', 'http://wortschatz.uni-leipzig.de/findlinks/')); - $this->register(new Spider('FindoryBot', 'Findroy', 'http://www.findory.com')); - $this->register(new Spider('Fish-Search-Robot', 'Fish search')); - $this->register(new Spider('Mozilla/4.0 (compatible: FDSE robot)', 'Fluid Dynamics')); - $this->register(new Spider('fouineur.9bit.qc.ca', 'Fouineur')); - $this->register(new Spider('Freecrawl', 'Freecrawl')); $this->register( new Spider( 'FreeWebMonitoring SiteChecker', @@ -218,27 +100,10 @@ public function __construct() 'https://www.freewebmonitoring.com/bot.html' ) ); - $this->register(new Spider('FreshpingBot', 'Freshping', 'https://freshping.io/')); $this->register(new Spider('FreshRSS', 'FreshRSS', 'https://freshrss.org')); - $this->register(new Spider('FunnelWeb', 'FunnelWeb')); - $this->register(new Spider('GaisBot', 'Gais', 'http://gais.cs.ccu.edu.tw/robot.php')); - $this->register(new Spider('gamekitbot', 'GAMEKIT', 'http://www.uchoose.de/crawler/gamekitbot/')); - $this->register(new Spider('gammaSpider', 'gammaSpider')); - $this->register(new Spider('gazz', 'gazz')); - $this->register(new Spider('gcreep', 'GCreep')); - $this->register(new Spider('genieBot', 'genieBot', 'http://64.5.245.11/faq/faq.html')); - $this->register(new Spider('geourl', 'GeoURL', 'http://geourl.org/bot.html')); - $this->register(new Spider('GetterroboPlus', 'GetterroboPlus Puu')); - $this->register(new Spider('GetURL.rexx', 'GetURL')); - $this->register(new Spider('Gigabot', 'Gigabot', 'http://www.gigablast.com/spider.html')); - $this->register(new Spider('Girafabot', 'Girafabot', 'http://www.girafa.com/')); - $this->register(new Spider('Goku', 'Goku', 'http://goku.ru/bot.htm; bot@goku.ru')); - $this->register(new Spider('Golem', 'Golem')); - $this->register(new Spider('gonzo', 'Gonzo')); $this->register(new Spider('Googlebot/', 'Google', 'http://www.google.com/bot.html')); $this->register(new Spider('Mediapartners-Google', 'Google AdSense', 'https://www.google.com/adsense/faq')); $this->register(new Spider('Googlebot-Image', 'Googlebot-Image', 'http://www.googlebot.com/bot.html')); - $this->register(new Spider('Googlebot-Mobile', 'Googlebot-Mobile', 'http://www.google.com/bot.html')); $this->register( new Spider( 'Googlebot-Video/1.0', @@ -279,169 +144,23 @@ public function __construct() ); $this->register(new Spider('GoogleOther', 'GoogleOther')); $this->register(new Spider('Google-Ads-Creatives-Assistant', 'Google-Ads-Creatives-Assistant')); - $this->register(new Spider('Google-AdWords-Express', 'Google-AdWords-Express')); $this->register(new Spider('AdsBot-Google', 'Google Ads-Bot', 'http://www.google.com/adsbot.html')); - $this->register( - new Spider('AdsBot-Google-Mobile', 'Google Ads-Bot Mobile', 'http://www.google.com/adsbot.html') - ); - $this->register(new Spider('Gpostbot', 'Gpostbot', 'http://www.gpost.info/help.php?c=bot')); - $this->register(new Spider('griffon', 'Griffon')); - $this->register(new Spider('Gromit', 'Gromit')); - $this->register(new Spider('http://grub.org', 'Grub Client')); - $this->register(new Spider('Gulper Web Bot', 'Gulper Bot')); - $this->register(new Spider('havIndex', 'havIndex')); - $this->register(new Spider('HeinrichderMiragoRobot', 'HeinrichderMiragoRobot')); - $this->register(new Spider('HenryTheMiragoRobot', 'HenryTheMiragoRobot')); $this->register(new Spider('HetrixTools Uptime Monitoring Bot', 'HetrixTools Uptime Monitoring Bot', 'https://hetrix.tools/uptime-monitoring-bot.html')); $this->register(new Spider('heritrix', 'Heritrix', 'https://github.com/internetarchive/heritrix3/wiki')); - $this->register(new Spider('HKU WWW Robot', 'HKU WWW Octopus')); - $this->register(new Spider('HolyCowDude', 'HolyCowDude', 'http://www.holycowdude.com/spider.htm')); - $this->register(new Spider('HomeTags', 'HomeTags', 'http://www.hometags.nl/bot')); - $this->register(new Spider('Hometown', 'Hometown')); - $this->register(new Spider('htdig', 'ht://Dig')); - $this->register(new Spider('AITCSRobot', 'HTML Index')); - $this->register(new Spider('HTMLgobble', 'HTMLgobble')); - $this->register(new Spider('I Robot', 'I, Robot')); - $this->register(new Spider('iajaBot', 'iajaBot')); - $this->register(new Spider('IBM_Planetwide', 'IBM_Planetwide')); - $this->register(new Spider('+http://www.icerocket.com/', 'IceRocket', 'http://www.icerocket.com/')); - $this->register(new Spider('ichiro', 'ichiro')); - $this->register( - new Spider('IlTrovatore-Setaccio', 'IlTrovatore-Setaccio', 'http://www.iltrovatore.it/aiuto/faq.html') - ); - $this->register(new Spider('image.kapsi.net', 'image.kapsi.net')); - $this->register(new Spider('Mozilla 3.01 PBWF (Win95)', 'Imagelock')); - $this->register(new Spider('IncyWincy', 'IncyWincy')); - $this->register(new Spider('infoobot', 'infoobot', 'https://www.infoo.nl/bot.html')); - $this->register(new Spider('Informant', 'Informant')); - $this->register(new Spider('InfoSeek Robot', 'InfoSeek Robot 1.0')); - $this->register(new Spider('Infoseek Sidewinder', 'Infoseek Sidewinder')); - $this->register(new Spider('InfoSpiders', 'InfoSpiders')); - $this->register(new Spider('INGRID', 'Ingrid')); - $this->register(new Spider('slurp@inktomi', 'Inktomi')); - $this->register(new Spider('Insitor', 'Insitor', 'http://www.insitor.de/')); - $this->register(new Spider('inspectorwww', 'Inspector Web')); - $this->register(new Spider('IAGENT', 'IntelliAgent')); - $this->register(new Spider('Intelliseek', 'Intelliseek', 'http://www.intelliseek.com/')); - $this->register(new Spider('Internet Cruiser Robot', 'Internet Cruiser')); - $this->register(new Spider('internetseer', 'Internet Seer')); - $this->register(new Spider('sharp-info-agent', 'Internet Shinchakubin')); - $this->register(new Spider('InternetLinkAgent', 'InternetLinkAgent')); - $this->register(new Spider('IRLbot', 'IRL Crawler', 'http://irl.cs.tamu.edu/crawler')); $this->register(new Spider('IonCrawl', 'IonCrawl', 'https://www.ionos.de/terms-gtc/faq-crawler-en')); - $this->register(new Spider('Iron33', 'Iron33')); - $this->register(new Spider('IsraeliSearch', 'Israeli-search')); - $this->register(new Spider('itchBot', 'itch')); - $this->register(new Spider('JavaBee', 'JavaBee')); - $this->register(new Spider('JBot', 'JBot')); - $this->register(new Spider('JCrawler', 'JCrawler')); - $this->register(new Spider('JetBot', 'JetEye', 'http://www.jeteye.com/jetbot.html')); - $this->register(new Spider('JoBo', 'JoBo')); - $this->register(new Spider('Jobot', 'Jobot')); - $this->register(new Spider('jobs.de', 'Jobs.de', 'http://www.jobs.de/')); - $this->register(new Spider('JoeBot', 'JoeBot')); - $this->register(new Spider('jumpstation', 'JumpStation')); - $this->register(new Spider('Katipo', 'Katipo')); - $this->register(new Spider('KDD-Explorer', 'KDD-Explorer')); - $this->register(new Spider('KIT-Fireball', 'KIT-Fireball')); - $this->register(new Spider('KO_Yappo_Robot', 'KO_Yappo_Robot')); - $this->register(new Spider('LabelGrab', 'LabelGrabber')); - $this->register(new Spider('larbin', 'larbin')); - $this->register(new Spider('legs', 'legs')); $this->register(new Spider('linkdexbot', 'Linkdex', 'http://www.linkdex.com/bots/')); - $this->register(new Spider('LinkScan Server', 'LinkScan')); - $this->register(new Spider('LinkWalker', 'LinkWalker')); - $this->register(new Spider('Linguee Bot', 'Linguee', 'http://www.linguee.com/bot')); - $this->register(new Spider('livedoorCheckers/', 'livedoorCheckers')); - $this->register(new Spider('Lockon', 'Lockon')); - $this->register(new Spider('logo.gif crawler', 'logo.gif')); - $this->register(new Spider('Lycos', 'Lycos')); $this->register(new Spider('Magpie', 'Magpie')); $this->register(new Spider('MJ12bot', 'Majestics MJ12bot')); - $this->register(new Spider('Mammoth', 'Mammoth', 'http://www.sli-systems.com')); - $this->register(new Spider('Marvin', 'Marvin')); - $this->register(new Spider('marvin/infoseek', 'marvin/infoseek')); - $this->register(new Spider('M/3.8', 'Mattie')); - $this->register(new Spider('MediaFox', 'MediaFox')); - $this->register( - new Spider('memorybot', 'Memorybot', 'http://archivethe.net/en/index.php/about/internet_memory1') - ); - $this->register(new Spider('mercator', 'Mercator', 'http://research.compaq.com/SRC/mercator/')); - $this->register(new Spider('MerzScope', 'MerzScope')); - $this->register(new Spider('METASpider', 'META', 'http://www.meta.com.ua/')); - $this->register(new Spider('MetaGer-LinkChecker', 'MetaGer')); - $this->register(new Spider('MindCrawler', 'MindCrawler')); - $this->register(new Spider('Miva', 'Miva')); - $this->register(new Spider('UdmSearch', 'mnoGoSearch')); - $this->register(new Spider('moget', 'moget')); - $this->register(new Spider('MOMspider', 'MOMspider')); - $this->register(new Spider('Monster', 'Monster')); - $this->register(new Spider('Moreoverbot', 'Moreover', 'http://www.moreover.com')); - $this->register(new Spider('msnbot', 'MSNBot', 'http://search.msn.com/msnbot.htm')); - $this->register(new Spider('MSRBOT', 'MSRBOT', 'http://research.microsoft.com/research/sv/msrbot/')); - $this->register(new Spider('MuscatFerret', 'Muscat Ferret')); - $this->register(new Spider('MwdSearch', 'Mwd.Search')); - $this->register(new Spider('NPBot', 'NameProtect')); - $this->register(new Spider('NaverBot', 'NaverBot', 'http://www.spidermatic.com/en/robot-spider/20')); - $this->register(new Spider('NDSpider', 'NDSpider')); - $this->register(new Spider('NEC-MeshExplorer', 'NEC-MeshExplorer')); - $this->register(new Spider('Nederland.zoek', 'Nederland.zoek')); - $this->register(new Spider('Neevabot', 'Neeva', 'https://neeva.com/neevabot')); - $this->register(new Spider('NerdyBot', 'NerdyBot', 'http://nerdybot.com/')); - $this->register(new Spider('NetCarta CyberPilot Pro', 'NetCarta WebMap')); + $this->register(new Spider('Yeti/', 'Naver Yeti', 'http://naver.me/spd')); $this->register(new Spider('Netcraft', 'Netcraft Web Server Survey', 'http://news.netcraft.com/')); $this->register(new Spider('Neticle Crawler', 'Neticle Crawler', 'https://neticle.com/bot/en/')); - $this->register(new Spider('NetMechanic', 'NetMechanic')); - $this->register(new Spider('NetScoop', 'NetScoop')); - $this->register(new Spider('newscan-online', 'newscan-online')); - $this->register( - new Spider('NextGenSearchBot 1', 'NextGenSearchBot', 'http://www.zoominfo.com/NextGenSearchBot') - ); - $this->register(new Spider('NHSEWalker', 'NHSE Web Forager')); - $this->register(new Spider('NIF', 'NIF', 'http://www.newsisfree.com/robot.php')); - $this->register(new Spider('NimbleCrawler', 'NimbleCrawler', 'http://www.healthline.com/aboutus.jsp')); - $this->register(new Spider('Nomad', 'Nomad')); - $this->register(new Spider('Norbert the Spider', 'Norbert', 'http://www.Burf.com')); - $this->register(new Spider('Gulliver', 'Northern Light')); - $this->register(new Spider('explorersearch', 'nzexplorer')); - $this->register(new Spider('Occam', 'Occam')); - $this->register(new Spider('Ocelli', 'Ocelli', 'http://www.globalspec.com/Ocelli')); - $this->register(new Spider('Online24-Bot', 'Online24-Bot')); - $this->register(new Spider('Openbot', 'Openbot', 'http://www.openfind.com.tw/robot.html')); - $this->register(new Spider('Openfind', 'Openfind data gatherer')); - $this->register(new Spider('Orbsearch', 'Orb Search')); - $this->register(new Spider('PackRat', 'Pack Rat')); - $this->register(new Spider('PageBoy', 'PageBoy')); $this->register(new Spider('PagePeeker', 'PagePeeker', 'https://pagepeeker.com/robots/')); $this->register(new Spider('Paqlebot', 'Pagle', 'http://www.paqle.dk/about/paqlebot')); $this->register(new Spider('Pandalytics', 'Pandalytics', 'https://domainsbot.com/pandalytics/')); - $this->register(new Spider('ParaSite', 'ParaSite')); - $this->register(new Spider('Patric', 'Patric')); - $this->register(new Spider('PEGASUS', 'pegasus')); - $this->register(new Spider('PerlCrawler/1.0 Xavatoria/2.0', 'PerlCrawler 1.0')); - $this->register(new Spider('PetalBot', 'PetalBot', 'https://aspiegel.com/petalbot')); - $this->register(new Spider('PGP-KA', 'PGP Key Agent')); - $this->register(new Spider('Duppies', 'Phantom')); - $this->register(new Spider('phpdig', 'PhpDig')); - $this->register(new Spider('PiltdownMan', 'PiltdownMan')); - $this->register(new Spider('Pimptrain\'s robot', 'Pimptrain.com\'s')); - $this->register(new Spider('pingalink', 'PingALink')); - $this->register(new Spider('Pioneer', 'Pioneer')); - $this->register(new Spider('PluckFeedCrawler', 'Pluck', 'http://www.pluck.com')); - $this->register(new Spider('PlumtreeWebAccessor', 'PlumtreeWebAccessor')); - $this->register(new Spider('PodNova', 'PodNova', 'http://www.podnova.com')); - $this->register(new Spider('Pompos', 'Pompos', 'http://dir.com/pompos.html')); - $this->register(new Spider('Poppi', 'Poppi')); - $this->register(new Spider('publiclibraryarchive.org', 'publiclibraryarchive.org')); - $this->register(new Spider('gestaltIconoclast', 'Popular Iconoclast')); - $this->register(new Spider('PortalJuice.com', 'Portal Juice')); - $this->register(new Spider('PortalBSpider', 'PortalB Spider')); + $this->register(new Spider('PetalBot', 'PetalBot', 'https://webmaster.petalsearch.com/site/petalbot')); $this->register( new Spider('Qualidator', 'Qualidator', 'www.qualidator.com/Web/de/Support/FAQ_OnlineTestStatistiken.htm') ); - $this->register(new Spider('www.kolinka.com', 'Project Kolinka Forum Search', 'http://www.kolinka.com/')); - $this->register(new Spider('psbot', 'psbot')); - $this->register(new Spider('Qango.com Web Directory', 'Qango', 'http://www.qango.com')); $this->register(new Spider('Qwant', 'Qwant', 'https://help.qwant.com/bot/')); $this->register( new Spider( @@ -451,196 +170,45 @@ public function __construct() ) ); $this->register(new Spider('SemrushBot', 'SemrushBot', 'http://semrush.com/bot/')); - $this->register(new Spider('StackRambler', 'Rambler', 'http://www.rambler.ru/')); - $this->register(new Spider('Raven', 'Raven Search')); - $this->register(new Spider('Resume Robot', 'Resume Robot')); - $this->register(new Spider('Road Runner: ImageScape Robot', 'Road Runner: The ImageScape Robot')); - $this->register(new Spider('RHCS', 'RoadHouse Crawling System')); - $this->register(new Spider('Robbie', 'Robbie the Robot')); - $this->register(new Spider('RoboCrawl', 'RoboCrawl')); - $this->register(new Spider('Robofox', 'RoboFox')); - $this->register(new Spider('Robot du CRIM 1.0a', 'Robot Francoroute')); - $this->register(new Spider('Robozilla', 'Robozilla')); - $this->register(new Spider('Roverbot', 'Roverbot')); - $this->register(new Spider('RSS-SPIDER', 'RSS Feed Seeker', 'http://www.rss-spider.com/fsb.php')); - $this->register(new Spider('RuLeS', 'RuLeS')); - $this->register(new Spider('RyzeCrawler', 'RyzeCrawler', 'http://www.domain2day.nl/crawler/')); - $this->register(new Spider('SafetyNet Robot', 'SafetyNet')); - $this->register(new Spider('SBIder', 'SBIder.', 'http://www.sitesell.com/sbider.html')); - $this->register(new Spider('Scharia', 'Scharia')); - $this->register(new Spider('Science-Index', 'Science-Index')); - $this->register(new Spider('Scooter', 'Scooter')); $this->register(new Spider('SearchAtlas', 'SearchAtlas', 'https://searchatlas.com/')); - $this->register(new Spider('SearchNZ', 'SearchNZ', 'http://www.searchnz.co.nz/')); - $this->register(new Spider('searchprocess', 'SearchProcess')); - $this->register( - new Spider('SearchmetricsBot', 'SearchmetricsBot', 'http://www.searchmetrics.com/en/searchmetrics-bot/') - ); - $this->register(new Spider('Seekbot', 'Seekbot', 'http://www.seekbot.net/bot.html')); $this->register(new Spider('SeekportBot', 'Seekport Bot', 'https://bot.seekport.com')); - $this->register(new Spider('Senrigan', 'Senrigan')); - $this->register(new Spider('Sensis Web Crawler', 'Sensis Web Crawler', 'http://www.sensis.com.au/help.do')); $this->register(new Spider('SentiBot', 'SentiBot', 'http://www.sentibot.eu')); - $this->register(new Spider('SEO Scanner', 'SEO Scanner')); $this->register(new Spider('SeobilityBot', 'SeobilityBot', 'https://www.seobility.net/sites/bot.html')); $this->register(new Spider('SEOkicks', 'SEOkicks', 'https://www.seokicks.de/robot.html')); - $this->register(new Spider('seostar.co', 'Seostar', 'https://seostar.co/robot/')); + $this->register(new Spider('serpstatbot', 'serpstatbot', 'http://serpstatbot.com/')); $this->register( - new Spider('SerendeputyBot', 'SerendeputyBot', 'http://serendeputy.com/about/serendeputy-bot') + new Spider('SeznamBot', 'Seznam Bot', 'https://o-seznam.cz/napoveda/vyhledavani/en/seznambot-crawler/') ); - $this->register(new Spider('serpstatbot', 'serpstatbot', 'http://serpstatbot.com/')); - $this->register(new Spider('SeznamBot/3.2', 'Seznam Bot', 'http://napoveda.seznam.cz/en/seznambot-intro/')); - $this->register(new Spider('SG-Scout', 'SG-Scout')); - $this->register(new Spider('Shagseeker', 'ShagSeeker')); - $this->register(new Spider('Shai\'Hulud', 'Shai\'Hulud')); - $this->register(new Spider('SimBot/1.0', 'Simmany Robot Ver1.0')); $this->register(new Spider('SimplePie', 'SimplePie', 'https://simplepie.org')); $this->register(new Spider('SkypeUriPreview', 'Skype Preview', 'https://www.skype.com/')); - $this->register(new Spider('ssearcher100', 'Site Searcher')); - $this->register(new Spider('Site Valet', 'Site Valet')); - $this->register(new Spider('http://www.site-list.net', 'Site-List', 'http://www.site-list.net')); - $this->register(new Spider('SiteTech-Rover', 'SiteTech-Rover')); - $this->register(new Spider('+SitiDi.net/SitiDiBot/', 'SitiDi.net/SitiDiBot')); - $this->register(new Spider('aWapClient', 'Skymob.com')); - $this->register(new Spider('Slack', 'Slackbot', 'https://api.slack.com/robots')); - $this->register(new Spider('SleepBot', 'SleepBot', 'http://sleepbot.com/')); - $this->register(new Spider('SLCrawler', 'SLCrawler')); - $this->register(new Spider('Sleek Spider', 'Sleek')); - $this->register(new Spider('ESISmartSpider', 'Smart Spider')); - $this->register(new Spider('Snapbot', 'Snapbot', 'http://www.snap.com/')); - $this->register(new Spider('Snooper', 'Snooper')); - $this->register(new Spider('sohu-search', 'sohu-search')); + $this->register(new Spider('Slackbot', 'Slackbot', 'https://api.slack.com/robots')); $this->register(new Spider('Sogou web spider', 'Sogou Web Spider', 'http://www.sogou.com/docs/help/webmasters.htm#07')); - $this->register(new Spider('Solbot', 'Solbot')); $this->register( new Spider('Speedy Spider', 'Speedy Spider', 'http://www.entireweb.com/about/search_tech/speedyspider/') ); - $this->register(new Spider('Sphere Scout', 'Sphere')); - $this->register(new Spider('Sphider2', 'Sphider')); - $this->register(new Spider('SpiderBot', 'SpiderBot')); - $this->register(new Spider('spiderline', 'Spiderline Crawler')); - $this->register(new Spider('SpiderMan', 'SpiderMan')); - $this->register(new Spider('SpiderView', 'SpiderView(tm)')); - $this->register(new Spider('mouse.house', 'spider_monkey')); - $this->register(new Spider('suke', 'Suke')); - $this->register(new Spider('suntek', 'suntek search engine')); $this->register(new Spider('Superfeedr', 'Superfeedr', 'http://superfeedr.com')); $this->register(new Spider('SurdotlyBot', 'SurdotlyBot', 'http://sur.ly/bot.html')); - $this->register(new Spider('Szukacz', 'Szukacz', 'http://www.szukacz.pl/html/RobotEnglishVersion.html')); - $this->register(new Spider('T-H-U-N-D-E-R-S-T-O-N-E', 'T-H-U-N-D-E-R-S-T-O-N-E')); - $this->register(new Spider('TinEye Crawler', 'TinEye', 'http://tineye.com/crawler.html')); + $this->register(new Spider('TinEye-bot', 'TinEye', 'https://tineye.com/crawler.html')); $this->register(new Spider('TikTokSpider', 'TikTokSpider', 'https://www.tiktok.com/about')); - $this->register(new Spider('Black Widow', 'TACH Black Widow')); $this->register(new Spider('Tapatalk CloudSearch', 'Tapatalk CloudSearch')); - $this->register(new Spider('Tarantula', 'Tarantula')); - $this->register(new Spider('tarspider', 'tarspider')); - $this->register(new Spider('dlw3robot', 'Tcl W3 Robot')); - $this->register(new Spider('TechBOT', 'TechBOT')); - $this->register(new Spider('Technoratibot', 'Technorati', 'http://technorati.com/about/')); - $this->register(new Spider('Templeton', 'Templeton')); - $this->register(new Spider('teoma', 'Teoma/Ask Jeeves', 'http://sp.teoma.com/docs/teoma/about/')); $this->register(new Spider('trovitBot', 'trovitBot', 'http://www.trovit.com/bot.html')); - $this->register(new Spider('JubiiRobot', 'The Jubii')); - $this->register(new Spider('NorthStar', 'The NorthStar Robot')); - $this->register(new Spider('w3index', 'The NWI Robot')); - $this->register(new Spider('Peregrinator-Mathematics', 'The Peregrinator')); - $this->register(new Spider('Pixray-Seeker', 'Pixray', 'http://www.pixray.com/pixraybot/')); $this->register( new Spider('TelegramBot (like TwitterBot)', 'TelegramBot (like TwitterBot)', 'https://telegram.org/') ); $this->register(new Spider('Testomatobot', 'TestomatoBot', 'https://www.testomato.com/bot')); - $this->register(new Spider('thumbshots-de-Bot', 'thumbshots-de-Bot')); - $this->register(new Spider('Thinkbot', 'Thinkbot')); - $this->register(new Spider('TITAN', 'TITAN')); - $this->register(new Spider('TitIn', 'TitIn')); - $this->register(new Spider('TLSpider', 'TLSpider')); - $this->register(new Spider('TMCrawler', 'TMCrawler')); $this->register(new Spider('trendictionbot', 'Trendiction-Bot', 'http://www.trendiction.com/bot')); - $this->register( - new Spider( - 'slysearch', - 'Turnitin.com', - 'http://www.turnitin.com/static/products_services/search_engines.html' - ) - ); $this->register(new Spider('TurnitinBot', 'TurnitinBot', 'http://www.turnitin.com/robot/crawlerinfo.html')); - $this->register(new Spider('TurtleScanner', 'Turtle', 'http://www.turtle.ru/')); - $this->register(new Spider('TwengaBot', 'Twenga', 'http://www.twenga.com/bot.html')); - $this->register(new Spider('Twiceler', 'Twiceler', 'http://www.cuill.com/twiceler/robot.html')); $this->register(new Spider('Twitterbot', 'Twitterbot', 'https://twitter.com/')); - $this->register(new Spider('UCSD-Crawler', 'UCSD Crawl')); - $this->register(new Spider('UMBC-memeta-Bot', 'UMBC')); - $this->register(new Spider('unisterbot', 'Unister')); - $this->register(new Spider('Unpartisan', 'Unpartisan', 'http://www.unpartisan.com')); $this->register(new Spider('Uptime-Kuma', 'Uptime-Kuma', 'https://uptime.kuma.pet/')); $this->register(new Spider('UptimeRobot/2.0', 'Uptime Robot', 'http://uptimerobot.com/')); - $this->register(new Spider('urlck', 'URL Check')); - $this->register(new Spider('URL Spider Pro', 'URL Spider Pro')); - $this->register(new Spider('Valkyrie', 'Valkyrie')); $this->register(new Spider('VelenPublicWebCrawler', 'Velen Crawler', 'https://velen.io')); - $this->register(new Spider('Verticrawl', 'Verticrawl')); - $this->register(new Spider('Victoria', 'Victoria')); - $this->register(new Spider('vision-search', 'vision-search')); - $this->register(new Spider('VoilaBot', 'VoilaBot', 'http://www.voila.com/')); - $this->register(new Spider('VisBot', 'VisBot', 'http://www.visvo.com/webmasters.html')); - $this->register(new Spider('Voyager', 'Voyager')); - $this->register(new Spider('VWbot_K', 'VWbot')); - $this->register(new Spider('W3M2', 'W3M2')); - $this->register(new Spider('w3mir', 'w3mir')); - $this->register(new Spider('w@pSpider', 'w@pSpider')); - $this->register(new Spider('appie', 'Walhello appie', 'http://www.robotstxt.org/db/appie.html')); - $this->register(new Spider('CrawlPaper', 'WallPaper')); - $this->register(new Spider('root', 'Web Core / Roots')); - $this->register(new Spider('WBSearchBot', 'Ware Bay', 'http://www.warebay.com/bot.html')); - $this->register(new Spider('WebMoose', 'Web Moose')); - $this->register(new Spider('WebBandit', 'WebBandit')); - $this->register(new Spider('WebCatcher', 'WebCatcher')); - $this->register(new Spider('Webclipping', 'Webclipping')); $this->register(new Spider('WebCopy', 'WebCopy')); - $this->register(new Spider('WebFetcher', 'webfetcher')); - $this->register(new Spider('weblayers', 'weblayers')); - $this->register(new Spider('WebLinker', 'WebLinker')); - $this->register(new Spider('wlm', 'Weblog Monitor')); - $this->register(new Spider('WebQuest', 'WebQuest')); - $this->register(new Spider('WebReaper', 'WebReaper')); - $this->register(new Spider('webs@recruit.co.jp', 'webs')); - $this->register(new Spider('websearchbench', 'WebSearchBench', 'http://websearchbench.cs.uni-dortmund.de/')); - $this->register(new Spider('WOLP', 'WebStolperer')); - $this->register(new Spider('webvac', 'WebVac')); - $this->register(new Spider('webwalk', 'webwalk')); - $this->register(new Spider('WebWalker', 'WebWalker')); - $this->register(new Spider('WebWatch', 'WebWatch')); - $this->register(new Spider('WebZinger', 'WebZinger')); - $this->register(new Spider('whatUseek_winona', 'whatUseek Winona')); - $this->register(new Spider('WhoWhere Robot', 'WebWatch', 'http://www.whowhere.com')); $this->register(new Spider('SurveyBot', 'Whois Source', 'http://www.whois.sc/info/webmasters/surveybot.html')); - $this->register(new Spider('Hazel\'s Ferret Web hopper', 'Wild Ferret Web Hopper')); - $this->register(new Spider('HRCrawler', 'HRCrawler')); $this->register(new Spider('WinHTTP', 'WinHTTP')); - $this->register(new Spider('wired-digital-newsbot', 'Wired Digital')); - $this->register(new Spider('zyborg', 'WiseNut')); $this->register(new Spider('WoltLabSuite', 'WoltLab Suite')); - $this->register(new Spider('OmniExplorer_Bot', 'WorldIndexer', 'http://www.omni-explorer.com')); - $this->register(new Spider('WWWC', 'WWWC')); - $this->register(new Spider('WWWeasel Robot', 'WWWeasel Robot')); - $this->register(new Spider('wwwster', 'wwwster')); - $this->register(new Spider('WWWWanderer', 'WWWWanderer')); - $this->register(new Spider('TECOMAC-Crawler', 'X-Crawler')); - $this->register(new Spider('XGET', 'XGET')); - $this->register(new Spider('cosmos', 'XYLEME Robot')); $this->register(new Spider('yacybot', 'YaCy-Bot', 'https://yacy.net/bot.html')); - $this->register(new Spider('YahooYSMcm', 'Yahoo Publisher Network', 'http://publisher.yahoo.com/')); - $this->register( - new Spider('Yahoo-Blogs', 'Yahoo-Blogs', 'http://help.yahoo.com/help/us/ysearch/crawling/crawling-02.html') - ); - $this->register(new Spider('Yahoo Pipes', 'Yahoo Pipes')); $this->register(new Spider('Yahoo! Slurp', 'Yahoo! Slurp', 'http://help.yahoo.com/help/us/ysearch/slurp')); - $this->register(new Spider('Yahoo-VerticalCrawler', 'Yahoo-VerticalCrawler')); - $this->register(new Spider('YahooFeedSeeker', 'YahooFeedSeeker', 'http://my.yahoo.com/s/publishers.html')); $this->register(new Spider('Yandex', 'Yandex', 'https://yandex.com/bots')); - $this->register(new Spider('zeus', 'Zeus Internet Marketing', 'http://www.cyber-robotics.com/')); - $this->register(new Spider('http://www.zorkk.com', 'Zork', 'http://www.zorkk.com')); - $this->register(new Spider('Zookabot', 'Zookabotk', 'http://zookabot.com/')); $this->register( new Spider('ZoominfoBot', 'ZoominfoBot', 'https://www.zoominfo.com/about-zoominfo/zoominfobot') ); @@ -652,8 +220,8 @@ public function __construct() $this->register(new Spider('Perplexity-User', 'Perplexity-User', 'https://perplexity.ai/perplexity-user')); $this->register( new Spider( - 'meta-webindexer', - 'Meta-WebIndexer', + 'meta-webindexer', + 'Meta-WebIndexer', 'https://developers.facebook.com/docs/sharing/webmasters/web-crawlers' ) );