codelists.spiderlist-phloc.xml Maven / Gradle / Ivy
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (C) 2006-2015 phloc systems
http://www.phloc.com
office[at]phloc[dot]com
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!--
If one of the following texts is contained in a user agent, it is considered to be a webspider.
Last update: 2012/10/01
-->
<list>
<item value="!Susie" />
<item value="'artviper" />
<item value="123pagesbot" />
<item value="123peoplebot" />
<item value="50.nu" />
<item value="A1 Sitemap Generator" />
<item value="ac-baidu.bot" />
<item value="AC-BaiduBot" />
<item value="AdMedia bot" />
<item value="AdsBot-Google" />
<item value="Advista Crawler" />
<item value="aiHitBot" />
<item value="askpeter_bot" />
<item value="asterias" />
<item value="Atomic5 Search" />
<item value="BackDoorBot" />
<item value="BacklinkCrawler" />
<item value="BackWeb" />
<item value="BaiduImagespider" />
<item value="Baiduspider" />
<item value="Bandit" />
<item value="BatchFTP" />
<item value="BeetleBot" />
<item value="betaBot" />
<item value="bingbot" />
<item value="BlackWidow" />
<item value="BlowFish" />
<item value="BotALot" />
<item value="Buddy" />
<item value="BuiltBotTough" />
<item value="BuiltWith" />
<item value="CamontSpider" />
<item value="CareerBot" />
<item value="CatchBot" />
<item value="CheeseBot" />
<item value="CherryPicker" />
<item value="ChinaClaw" />
<item value="Cityreview Robot" />
<item value="CMS Crawler" />
<item value="Collector" />
<item value="Comodo-Certificates-Spider" />
<item value="Content Crawler" />
<item value="contype" />
<item value="Copier" />
<item value="CopyRightCheck" />
<item value="cosmos" />
<item value="[email protected]" />
<item value="crawler4j" />
<item value="CrawlerBoy" />
<item value="Crescent" />
<item value="Custo" />
<item value="DBLBot" />
<item value="DCPbot" />
<item value="dcrawl" />
<item value="Declumbot" />
<item value="Die Suchmaschine der Technik" />
<item value="DISCoPump" />
<item value="discoverybot" />
<item value="DKIMRepBot" />
<item value="dlcbot" />
<item value="DNS-Digger-Explorer" />
<item value="DomainCrawler" />
<item value="dombot" />
<item value="DoubleVerify Crawler" />
<item value="Download Demon" />
<item value="DownloadWonder" />
<item value="Downloader" />
<item value="Drip" />
<item value="DTS Agent" />
<item value="eCatch" />
<item value="EdisterBot" />
<item value="ee://aol/http" />
<item value="EirGrabber" />
<item value="EmailSiphon" />
<item value="EmailWolf" />
<item value="ERIPPbot" />
<item value="Eurobot" />
<item value="eventax" />
<item value="Exositesbot" />
<item value="Express WebPictures" />
<item value="ExtractorPro" />
<item value="EyeNetIE" />
<item value="fastbot" />
<item value="favicon Internet survey research" />
<item value="EdisterBot" />
<item value="FeedBot" />
<item value="FNRBot" />
<item value="FSPBot" />
<item value="FyberSpider/" />
<item value="GarlikCrawler" />
<item value="GSLFbot" />
<item value="Generalbot" />
<item value="GetSmart" />
<item value="GetWeb!" />
<item value="GG PeekBot" />
<item value="Go!Zilla" />
<item value="Go-Ahead-Got-It" />
<item value="gold crawler" />
<item value="Google Bot" />
<item value="Google Keyword Generator" />
<item value="GoogleDocs" />
<item value="GoogleToolbar" />
<item value="Gorila.ro crawler" />
<item value="GornKer" />
<item value="gotit" />
<item value="Grabber" />
<item value="GrabNet" />
<item value="Grafula" />
<item value="GrepNetstat" />
<item value="gsa-crawler" />
<item value="GSLFbot" />
<item value="Herold" />
<item value="HMView" />
<item value="HomeTags" />
<item value="hpa-bot" />
<item value="HTMLCodeSearchBot" />
<item value="HTTrack" />
<item value="http://Anonymouse.org/" />
<item value="http://bot.sjn.com" />
<item value="http://SiteIntel.net Bot" />
<item value="http://wmtips.com/tools/" />
<item value="http://www.amagit.com/" />
<item value="http://www.forumseek.net/ BOT" />
<item value="http://www.lssw365.net/" />
<item value="http://www.web.nl/webmasters/spider.html" />
<item value="HuaweiSymantecSpider" />
<item value="humanlinks" />
<item value="i-market-bot" />
<item value="ia_archiver" />
<item value="ibot" />
<item value="iCCrawler" />
<item value="ichiro" />
<item value="Image Stripper" />
<item value="Image Sucker" />
<item value="Infohelfer" />
<item value="InfoNaviRobot" />
<item value="InsurancoBot" />
<item value="IntelCSbot" />
<item value="intelium_bot" />
<item value="InterGET" />
<item value="Internet Ninja" />
<item value="IPTCBOT" />
<item value="Iria" />
<item value="jabubo.com crawler" />
<item value="JennyBot" />
<item value="JetCar" />
<item value="Jetslide" />
<item value="JikeSpider" />
<item value="JoobleBot" />
<item value="JOC Web Spider" />
<item value="Jyxobot" />
<item value="KaloogaBot" />
<item value="KeepRightBot" />
<item value="Kenjin.Spider" />
<item value="KIT webcrawler" />
<item value="lftp" />
<item value="likse" />
<item value="Linguee Bot" />
<item value="LinksCrawler" />
<item value="LinkextractorPro" />
<item value="Linkman" />
<item value="LinkScan" />
<item value="LinkStash" />
<item value="LinkWalker" />
<item value="lssbot" />
<item value="LSSRocketCrawler" />
<item value="ltbot" />
<item value="m-crawler" />
<item value="Magnet" />
<item value="Mag-Net" />
<item value="Mass Downloader" />
<item value="Memo" />
<item value="MetaGeneratorCrawler" />
<item value="metager2-verification-bot" />
<item value="MIDown tool" />
<item value="Mirror" />
<item value="Mister PiX" />
<item value="Moose" />
<item value="Mozilla 5.0/Bot" />
<item value="MSR-ISRCCrawler" />
<item value="mylinkcheck" />
<item value="mysmutsearch-crawler" />
<item value="Navroad" />
<item value="nb-bot" />
<item value="NearSite" />
<item value="Net Vampire" />
<item value="NetAnts" />
<item value="netEstate NE Crawler" />
<item value="netEstate RSS crawler" />
<item value="neticle Bot" />
<item value="Neticle Crawler" />
<item value="NetSeer crawler" />
<item value="NetSpider" />
<item value="NetZIP" />
<item value="news bot" />
<item value="nu_tch-princeton" />
<item value="Nutscrape" />
<item value="oBot" />
<item value="oegp" />
<item value="Offline Explorer" />
<item value="Offline Navigator" />
<item value="Openfind" />
<item value="OpenindexSpider" />
<item value="OSS-bot" />
<item value="oxado-bot" />
<item value="PageGrabber" />
<item value="Papa Foto" />
<item value="PaperLiBot" />
<item value="pavuk" />
<item value="pcBrowser" />
<item value="Phantom.js bot" />
<item value="PHarvester" />
<item value="PHPCrawl" />
<item value="Pixray-Seeker" />
<item value="PiplBot" />
<item value="Plagiat Web Spider" />
<item value="pmoz.info ODP link checker" />
<item value="Pockey" />
<item value="PopScreenBot" />
<item value="PostPost" />
<item value="PrivacyFinder" />
<item value="ProCogBot" />
<item value="ProductoDownloadUrlBot" />
<item value="ProWebWalker" />
<item value="proximic" />
<item value="Pump" />
<item value="Punk Spider" />
<item value="Purebot" />
<item value="Qualidator.com Bot" />
<item value="QueryN.Metasearch" />
<item value="RealDownload" />
<item value="Reaper" />
<item value="Recorder" />
<item value="ReGet" />
<item value="rdfbot" />
<item value="RMA" />
<item value="rogerbot" />
<item value="robotgenius" />
<item value="RTGI" />
<item value="Scrapy" />
<item value="seexie.com_bot" />
<item value="Semager" />
<item value="semantissimo.de" />
<item value="SEOENGWorldBot" />
<item value="Seznam" />
<item value="ShowyouBot" />
<item value="sindice-fetcher" />
<item value="sindice-site-manager" />
<item value="Siphon" />
<item value="SISTRIX Crawler" />
<item value="SiteSnagger" />
<item value="SiteSucker" />
<item value="SmartDownload" />
<item value="Snake" />
<item value="SpaceBison" />
<item value="spbot" />
<item value="SpellCheck Bot" />
<item value="Sphider" />
<item value="SpiderLing" />
<item value="Spider.TerraNautic.net" />
<item value="ssearch_bot" />
<item value="Steeler" />
<item value="Stripper" />
<item value="Sucker" />
<item value="suggybot" />
<item value="suma spider" />
<item value="SuperBot" />
<item value="SuperHTTP" />
<item value="Surfbot" />
<item value="SWAT Crawler" />
<item value="SWEBot" />
<item value="SygolBot" />
<item value="textdigger.com" />
<item value="ThumbShots-Bot" />
<item value="tlap.hu bot" />
<item value="Toplistbot" />
<item value="TripAdvisorBot" />
<item value="TweetedTimes Bot" />
<item value="TwengaBot" />
<item value="ua-bot" />
<item value="UCMore Crawler" />
<item value="UnisterBot" />
<item value="Updownerbot" />
<item value="voodoo-it" />
<item value="WBSearchBot" />
<item value="Web Crawler" />
<item value="Web Image Collector" />
<item value="Web Sucker" />
<item value="WebAuto" />
<item value="WebCapture" />
<item value="WebCompanyCrawler" />
<item value="WebCopier" />
<item value="WebDataCentreBot" />
<item value="WebFetch" />
<item value="WebGo IS" />
<item value="webinatorbot" />
<item value="WebLeacher" />
<item value="WebmasterCoffee" />
<item value="WebReaper" />
<item value="WebRipper" />
<item value="WebSauger" />
<item value="Website eXtractor" />
<item value="Website Quester" />
<item value="WebStripper" />
<item value="WebWhacker" />
<item value="WebZIP" />
<item value="Wer-liefert-was Crawler" />
<item value="wf_crawler" />
<item value="Wget" />
<item value="Whacker" />
<item value="Whoismindbot" />
<item value="Widow" />
<item value="wikiwix-bot" />
<item value="Wotbox" />
<item value="wsAnalyzer" />
<item value="WWW-Mechanize" />
<item value="www.integromedb.org/Crawler" />
<item value="www.sameid.net" />
<item value="WWWOFFLE" />
<item value="Xaldon WebSpider" />
<item value="XmarksFetch" />
<item value="Y!J-BRW" />
<item value="yacybot" />
<item value="Yahoo! SearchMonkey" />
<item value="Yahoo-MMCrawler" />
<item value="YahooCacheSystem" />
<item value="YahooExternalCache" />
<item value="yCrawl" />
<item value="your-search-bot" />
<item value="yovadis" />
<item value="ZillaCrawler" />
<item value="ZipppBot" />
<item value="ZoomSpider" />
<item value="ZrobZakupyBot" />
<item value="ZSEBOT" />
</list> © 2015 - 2025 Weber Informatics LLC | Privacy Policy