############################### # robots.txt file for this website # addresses all robots by using wild card * # list folders robots are not allowed to index Crawl-delay: 40 User-agent: * Allow: /best-of-the-best.html Disallow: /pdfs/ Disallow: /top/ Disallow: /marion.html/ Disallow: /HCBOE.html/ Disallow: /osceola.html/ # End of robots.txt file # ###############################