#START: User-Agent: *
User-Agent: *
#start: disallow directory indexing
Disallow: /bpre/
Disallow: /common/css/
Disallow: /common/images/
Disallow: /common/jscript/
Disallow: /common/php/
Disallow: /mail/
Disallow: /rss/
Disallow: /site/
#end: disallow directory indexing
Sitemap: http://hotel.wbweb.co.uk/sitemap.xml
#start: disallow file indexing
Disallow: /favicon.ico
Disallow: /hotel.rss
#end: disallow file indexing
#END: User-Agent: *
#START: User-Agent: Sphider2
User-Agent: Sphider2
#start: disallow directory indexing
Disallow: /bpre/
Disallow: /common/css/
Disallow: /common/images/
Disallow: /common/jscript/
Disallow: /common/php/
Disallow: /mail/
Disallow: /rss/
Disallow: /site/
#end: disallow directory indexing
Sitemap: http://hotel.wbweb.co.uk/sitemap.xml
#start: disallow file indexing
Disallow: /favicon.ico
Disallow: /hotel.rss
#end: disallow file indexing
#END: User-Agent: Sphider2