# /robots.txt - указывает поисковым роботам какие каталоги
#               брать для индексации не следует.
#               Если пуст или не существует, то все можно брать
#
#     Пример /robots.txt file
#
#     User-agent: webcrawler
#     Disallow:
#
     User-agent: *
     Disallow: /archive
     Disallow: /guest/cgi-bin
     Disallow: /guest/gb.txt
     Disallow: /images
     Disallow: /mystery
     Disallow: /news
     Disallow: /newsline
     Disallow: /oz/book.txt
     Disallow: /oz/book.txt.prev
     Disallow: /oz/readme
     Disallow: /postcards/cards
     Disallow: /postcards/cgi-bin
     Disallow: /postcards/image
     Disallow: /railroad
     Disallow: /region/description
     Disallow: /region/images
     Disallow: /religion/old
     Disallow: /survey/cgi-bin
     Disallow: /survey/lac
     Disallow: /survey/result
     Disallow: /vom
     Disallow: /foot.php
     Disallow: /forbid.shtml
     Disallow: /header.php
     Disallow: /header2.php
     Disallow: /style.css