# /robots.txt file for http://www.midfirst.com/

# SortSite
User-agent: PowerMapper
allow: /


# exabot.com
#	Mozilla/5.0+(compatible;+Exabot/3.0;++http://www.exabot.com/go/robot)
#	http://www.exabot.com/go/robot
User-agent: exabot
Disallow: /

# Riddler
#	Riddler (http://riddler.io/about)
#	http://riddler.io/about
User-agent: Riddler
Disallow: /


# SMTBot
#	http://www.similartech.com/smtbot
#	Mozilla/5.0+(compatible;+SMTBot/1.0;++http://www.similartech.com/smtbot)
User-agent: SMTBot
Disallow: /


# Majestic Bot (MJ12bot)
#	http://www.majestic12.co.uk/bot.php
#	Mozilla/5.0+(compatible;+MJ12bot/v1.4.5;+http://www.majestic12.co.uk/bot.php?+)
User-agent: MJ12bot
Disallow: /


# SeeBot
#	http://www.seegnify.com/bot
#	seebot/2.0 (+http://www.seegnify.com/bot)
User-agent: SeeBot
Disallow: /


# Archive.org Bot
#	http://archive.org/details/archive.org_bot
#	Mozilla/5.0+(compatible;+archive.org_bot;+Wayback+Machine+Live+Record;++http://archive.org/details/archive.org_bot)
#	Mozilla/5.0+(compatible;+special_archiver/3.1.1++http://www.archive.org/details/archive.org_bot)
User-agent: archive.org_bot
Disallow: /


# AdBeat Bot
#	https://www.adbeat.com/operation_policy
#	adbeat_bot
User-agent: adbeat_bot
Disallow: /


# MeanPathBot
#	http://www.meanpath.com/meanpathbot.html
#	Mozilla/5.0+(compatible;+meanpathbot/1.0;++http://www.meanpath.com/meanpathbot.html)
User-agent: meanpathbot
Disallow: /


# Mojeek Bot
#	https://www.mojeek.com/bot.html
#	Mozilla/5.0+(compatible;+MojeekBot/0.6;++https://www.mojeek.com/bot.html)
User-agent: MojeekBot
Disallow: /


# Speedy Spider / EntireWeb Search
#	http://www.entireweb.com
#	Speedy+Spider+(http://www.entireweb.com)
User-agent: Speedy Spider
Disallow: /


# GenieO
#	http://www.genieo.com/webfilter.html
#	Mozilla/5.0+(compatible;+Genieo/1.0+http://www.genieo.com/webfilter.html)
User-agent: genieo
Disallow: /


# scrapy.org
# by default ROBOTSTXT_OBEY is set to false 
#	http://doc.scrapy.org/en/latest/topics/settings.html
User-agent: Scrapy
Disallow: /


# Gigablast Open Source Search Engine/Crawler/Spider
# You can ignore robots.txt
#	http://gigablast.com/admin.html
#	GigablastOpenSource/1.0
User-agent: GigablastOpenSource
Disallow: /


# Proximic
#	http://www.proximic.com/info/spider.php
#	Mozilla/5.0+(compatible;+proximic;++http://www.proximic.com/info/spider.php)
User-agent: proximic
Disallow: /


# AppleBot
#	Used by Siri/Spotlight suggestions (commented out for now)
#	http://www.apple.com/go/applebot
#	Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_10_1)+AppleWebKit/600.2.5+(KHTML,+like+Gecko)+Version/8.0.2+Safari/600.2.5+(Applebot/0.1;++http://www.apple.com/go/applebot)
#	Mozilla/5.0+(iPhone;+CPU+iPhone+OS+8_1+like+Mac+OS+X)+AppleWebKit/600.1.4+(KHTML,+like+Gecko)+Version/8.0+Mobile/12B410+Safari/600.1.4+(Applebot/0.1;++http://www.apple.com/go/applebot)
#	Mozilla/5.0+(compatible;+Applebot/0.3;++http://www.apple.com/go/applebot)
#User-agent: applebot
#Disallow: /


# CCBot
#	http://commoncrawl.org/faq/
#	CCBot/2.0+(http://commoncrawl.org/faq/)
User-agent: CCBot
Disallow: /

# Plukkie
#	http://www.botje.com/plukkie.htm
#	Mozilla/5.0+(compatible;+Plukkie/1.5;+http://www.botje.com/plukkie.htm)
User-agent: Plukkie
Disallow: /


# Lipperhey Kaus Australis
#	https://www.lipperhey.com/en/about/
#	Mozilla/5.0+(compatible;+Lipperhey-Kaus-Australis/5.0;++https://www.lipperhey.com/en/about/)
User-agent: lipperhey
Disallow: /


# oBot
#	http://filterdb.iss.net/crawler/
#	Mozilla/5.0+(compatible;+oBot/2.3.1;+http://filterdb.iss.net/crawler/)
#	Mozilla/5.0+(compatible;+oBot/2.3.1;++http://filterdb.iss.net/crawler/)
User-agent: oBot
Disallow: /


# Screaming Frog SEO Spider
#	Downloaded software, robots.txt can be ignored
#	http://www.screamingfrog.co.uk/seo-spider/user-guide/general/
#	Screaming+Frog+SEO+Spider/2.55
#	Screaming+Frog+SEO+Spider/3.3
User-agent: Screaming Frog SEO Spider
Disallow: /


# VegeBot
#	http://www.exactbot.com/vegebot/index.html
User-agent: VegeBot
Disallow: /


# ScreenerBot
#	ScreenerBot Crawler Beta 2.0 (+http://www.ScreenerBot.com)
User-agent: ScreenerBot
Disallow: /


# Garlik from Experian
#	GarlikCrawler/1.2 (http://garlik.com/, crawler@garlik.com)
User-agent: Garlik
Disallow: /


#	http-kit/2.0

#	BusinessBot: Nathan@lead-caddy.com
User-agent: BusinessBot
Disallow: /


#	FAST Enterprise Crawler/5.3.4 (crawler@fast.no)
User-agent: FAST Enterprise
Disallow: /


# Nutch
#	http://nutch.apache.org/bot.html
#	Nutch+Spider/Nutch-1.5
#	mycrowl/Nutch-1.9
#	nutch+crawler/Nutch-1.9
#	sky+nutch+crawler/Nutch-1.9
#	Kraken/Nutch-2.2.1+(Nutch+crawler+launched+by+Integral+Ad+Science,+Inc.;+TBD;+TBD)
User-agent: Nutch
Disallow: /


# Open Link Profiler
#	Mozilla/5.0+(compatible;+spbot/4.4.2;++http://OpenLinkProfiler.org/bot+)
#	http://OpenLinkProfiler.org/bot
User-agent: spbot
Disallow: /


# James Bot
#	Mozilla/5.0+(Windows;+U;+Windows+NT+5.1;+en-US;+rv:1.8.1.6)+Gecko/20070725+Firefox/2.0.0.6+-+James+BOT+-+WebCrawler+http://cognitiveseo.com/bot.html
#	http://cognitiveseo.com/bot.html
User-agent: JamesBOT
Disallow: /


# Qwantify
#	Mozilla/5.0+(compatible;+Qwantify/2.1n;++https://www.qwant.com/)/*
#	Mozilla/5.0+(compatible;+Qwantify/2.0n;++https://www.qwant.com/)/*
#	https://www.qwant.com/



# Kraken
#	Mozilla/5.0+(compatible;+Kraken/0.1;+http://linkfluence.net/;+bot@linkfluence.net)
#	http://linkfluence.net/
User-agent: Kraken
Disallow: /


#	Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10.9;+rv:28.0)+Gecko/20100101+Firefox/28.0+(FlipboardProxy/1.1;++http://flipboard.com/browserproxy)
#	http://flipboard.com/browserproxy
User-agent: Flipboard
Disallow: /


#	Mozilla/5.0+(compatible;+linkdexbot/2.0;++http://www.linkdex.com/bots/)
#	http://www.linkdex.com/bots/
User-agent: linkdex
Disallow: /


#	Mozilla/5.0+(compatible;+MSIE+10.0;+Windows+NT+6.1;+Trident/6.0)+SiteCheck-sitecrawl+by+Siteimprove.com
#	Siteimprove.com
User-agent: SiteCheck-sitecrawl by Siteimprove.com
Disallow: /


#	Mozilla/5.0+(compatible;+MegaIndex.ru/2.0;++http://megaindex.com/crawler)
#	http://megaindex.com/crawler
User-agent: MegaIndex
Disallow: /


#	Prlog
#	Mozilla/5.0+(compatible;+Prlog/1.0;++http://prlog.ru/)
#	http://prlog.ru/
User-agent: Prlog
Disallow: /

#	Mozilla/5.0+(compatible;+aiHitBot/2.9;++https://www.aihitdata.com/about)
#	https://www.aihitdata.com/about
User-agent: AiHitBot
Disallow: /


#	KinglandSystemsCorp/KinglandSystemsCorp-crawler-2.0.1+(A+prototype+nutch+crawler+configuration+from+Kingland+Systems;+http://www.kingland.com;+kyle+at+kingland+dot+com)
#	http://www.kingland.com
User-agent: KinglandSystemsCorp
Disallow: /


#	Mozilla/5.0+(compatible;+SitemapBot/0.4;++http://www.SitemapBot.com;+SitemapBot@stmp.com)
#	http://www.sitemapbot.com
User-agent: SitemapBot
Disallow: /


#	Mozilla/5.0+(compatible;+Googlebot/2.1;+https://www.deepcrawl.com/bot)
#	https://www.deepcrawl.com/bot
User-agent: deepcrawl
Disallow: /


#	Mozilla/5.0+(Windows+NT+6.1;+WOW64;+rv:38.0)+Gecko/20100101+Firefox/38.0+AlexaToolbar/alxf-2.21
#	http://www.alexa.com/toolbar
#User-agent: AlexaToolbar
#Disallow: /



#	Mozilla/5.0+(compatible;+Yeti/1.1;++http://help.naver.com/robots/)
#	http://help.naver.com/robots/
User-agent: Yeti
Disallow: /


#	Mozilla/5.0+(Windows+NT+6.3;+WOW64;+rv:36.0)+Gecko/20100101+Firefox/36.0+(NetShelter+ContentScan,+contact+abuse@inpwrd.com+for+information)
User-agent: NetShelter
Disallow: /

#	VSE/1.0+(rabraham@multiview.com)

#	LSSRocketCrawler/1.0 LightspeedSystems
#	http://lightspeedsystems.com
User-agent: LSSRocketCrawler
Disallow: /


#	Mozilla/5.0+(compatible;+idmarch+Automatic.beta/1.3;++http://www.idmarch.org/bot.html)
#	http://www.idmarch.org/bot.html
User-agent: idmarch
Disallow: /



#	Mozilla/5.0+(compatible;+CukBot;+Not+a+spammer;+++https://www.companiesintheuk.co.uk/bot.html)
#	https://www.companiesintheuk.co.uk/bot.html
User-agent: CukBot
Disallow: /


#	Mozilla/5.0+(compatible;+ExpertSearchSpider++http://www.expertsearch.nl/spider)
#	http://www.expertsearch.nl/spider
User-agent: ExpertSearchSpider
Disallow: /


#	Mozilla/5.0+(compatible;+Goodzer/2.0;+crawler@goodzer.com)
User-agent: Goodzer
Disallow: /


#	WeSEE
#	http://www.wesee.com/
User-agent: WeSEE
Disallow: /



#	Symfony2+BrowserKit
User-agent: Symfony2
Disallow: /

#	rogerbot/1.0+(http://moz.com/help/pro/what-is-rogerbot-,+rogerbot-crawler+shiny@moz.com)
#	http://moz.com/help/pro/what-is-rogerbot-
user-agent: rogerbot
allow: /
disallow: /assets
disallow: /media
disallow: /images
disallow: /bundles
disallow: /App_Images
disallow: /locations
sitemap: sitemap.xml



#	bitlybot
#	http://bit.ly ??
User-agent: bitlybot
Disallow: /


#	Mozilla/5.0+(Windows;+U;+Windows+NT+6.0;+en-GB;+rv:1.0;+trendictionbot0.5.0;+trendiction+search;+http://www.trendiction.de/bot;+please+let+us+know+of+any+problems;+web+at+trendiction.com)+Gecko/20071127+Firefox/3.0.0.11
#	http://www.trendiction.de/bot
User-agent: trendictionbot
Disallow: /


#	SearchMassive+internal+links+crawler/Nutch-1.10+(Crawl+and+collect+internal+links+from+websites)
User-agent: SearchMassive
Disallow: /


#	spiderbot
User-agent: spiderbot
Disallow: /


#	yacybot+(/global;+amd64+Linux+3.16.0-4-amd64;+java+1.7.0_79;+Europe/de)+http://yacy.net/bot.html
#	yacybot+(/global;+amd64+Linux+3.16.0-4-amd64;+java+1.7.0_79;+America/en)+http://yacy.net/bot.html
User-agent: yacybot
Disallow: /


#	AboutUsBot/Harpy+(Website+Analysis;+http://www.aboutus.org/Aboutus:Bot;+help@aboutus.org)
#	http://www.aboutus.org/Aboutus:Bot
User-agent: AboutUsBot
Disallow: /


#	http://www.nominet.org.uk/privacypolicy


#	Mozilla/5.0+(compatible;+DotBot/1.1;+http://www.opensiteexplorer.org/dotbot,+help@moz.com)
#	http://www.opensiteexplorer.org/dotbot
user-agent: DotBot
allow: /
disallow: /assets
disallow: /media
disallow: /images
disallow: /bundles
disallow: /App_Images
disallow: /locations
sitemap: sitemap.xml


# BrightEdge
user-agent: BrightEdge
disallow: /


#	Perl-Win32::Internet/0.087


#	Twitterbot/1.0


#	Xenu Link Sleuth 1.3.4
#	Xenu Link Sleuth/1.3.8
User-agent: Xenu Link Sleuth
Disallow: /


# Go language
#	Go+1.1+package+http
User-agent: Go 1.1
Disallow: /


# wotbox.com
#	http://www.wotbox.com/bot/
#	Wotbox/2.01+(+http://www.wotbox.com/bot/)
User-agent: Wotbox
Disallow: /

# yandex
#	http://yandex.com/bots
#	Mozilla/5.0+(compatible;+YandexBot/3.0;++http://yandex.com/bots)
User-agent: YandexBot
Disallow: /


#	Mozilla/5.0+(compatible;+Yahoo!+Slurp;+http://help.yahoo.com/help/us/ysearch/slurp)
#	http://help.yahoo.com/help/us/ysearch/slurp
# User-agent: Yahoo! Slurp
# Disallow: /


# Yahoo Japan
#	http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/
#	Y!J-ASR/0.1+crawler+(http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)
User-agent: Y!J-ASR
Disallow: /



# crazywebcrawler.com
User-agent: CrazyWebCrawler-Spider
Disallow: /

# crazywebcrawler.com
User-agent: CrazyWebCrawler
Disallow: /

# Dow Jones Searchbot
#	Mozilla/5.0+(compatible;+Dow+Jones+Searchbot)
User-agent: Dow Jones SearchBot
Disallow: /


# WeCrawlForThePeace - We are not Evil
User-agent: WeCrawlForThePeace
Disallow: /

# WeLikeLinks - WeAreNotEvil
User-agent: WeLikeLinks
Disallow: /

# icevikatam/1.0
#User-agent: icevikatam
#Disallow: /


# Toweya search engine (FR)
#	Toweyabot: toweya.com
User-agent: Toweyabot
Disallow: /


#	TScoutBot 1.0
User-agent: TScoutBot
Disallow: /


# Wappalyzer
#	Chrome/49.0.2623.87 (compatible; Wappalyzer; +https://github.com/AliasIO/Wappalyzer)
User-agent: Wappalyzer
Disallow: /


# standard users
User-agent: *
Disallow: /*.axd?
Disallow: /*.axd
#Disallow: /*?
#Disallow: /*?*
#Disallow: /*&*
Disallow: /*:*
Disallow: /*?*'*
Disallow: /page-not-found
Disallow: /general-error
Disallow: /cookies-not-enabled
Disallow: /maintenance-page
Disallow: /unsupported-browser
Disallow: /complete
Disallow: /online-banking-return/
Disallow: /search/
Disallow: /splashpages/*
Disallow: /App_Images/*
Disallow: /assets/*
Disallow: /media/*
Disallow: /documents/*
Disallow: /locations/select-location
Disallow: /locations/directions?*
Disallow: /locations/temporary-operating-hours
Allow: /locations/map-detail?*
SiteMap: https://www.midfirst.com/SiteMap.xml