НАСТРОЙКИ / ROBOTS.TXT для HOSTCMS

User-agent: *
Disallow: /admin
Disallow: /search
Disallow: /templates
Disallow: /captcha.php
Disallow: /403
Disallow: /404
Disallow: /*filter=*
Disallow: /*cart/*
Disallow: /*search/*
Disallow: /*sorting=*
Disallow: /*?comments=*
Disallow: /*?producer_id=*
Disallow: /*?on_page=*
Disallow: /showbanner/
Allow: /hostcmsfiles/css/*
Allow: /hostcmsfiles/js/*

User-agent: Yandex
Crawl-delay: 50

Host: site.ru
Sitemap: http://site.ru/sitemap.xml

User-agent: Yandex
Disallow: /admin
Disallow: /search
Disallow: /templates
Disallow: /showbanner
Disallow: /captcha.php
Disallow: /403
Disallow: /404
Disallow: /users/
Disallow: /*tag
Disallow: /*page
Disallow: */page/
Disallow: /compare_items/
Disallow: /*?
Disallow: /*filter=*
Disallow: /*cart/*
Disallow: /*sorting=*
Disallow: /*?comments=*
Disallow: /*?producer_id=*
Disallow: /*?on_page=*
Disallow: /*?text=*
Disallow: *.pdf
Disallow: *.xls
Disallow: *.doc
Disallow: *.ppt
Disallow: *.txt
Allow: /hostcmsfiles/css/*
Allow: /hostcmsfiles/js/*
Allow: /hostcmsfiles/jquery/*
Allow: */upload/*.jpg
Allow: */upload/*.jpeg
Allow: */upload/*.png
Allow: */upload/*.gif
Clean-param: sort
Clean-param: utm_source&utm_medium&utm_term&utm_content&utm_campaign&yclid&gclid&_openstat&from /
Clean-param: openstat

User-agent: Googlebot
Disallow: /admin
Disallow: /search
Disallow: /templates
Disallow: /showbanner
Disallow: /captcha.php
Disallow: /403
Disallow: /404
Disallow: /users/
Disallow: /*tag
Disallow: /*page
Disallow: */page/
Disallow: /compare_items/
Disallow: /*?
Disallow: /*filter=*
Disallow: /*cart/*
Disallow: /*sorting=*
Disallow: /*?comments=*
Disallow: /*?producer_id=*
Disallow: /*?on_page=*
Disallow: /*?text=*
Disallow: *.pdf
Disallow: *.xls
Disallow: *.doc
Disallow: *.ppt
Disallow: *.txt
Disallow: *utm
Disallow: *clid=
Disallow: *openstat
Disallow: *from
Allow: /hostcmsfiles/css/*
Allow: /hostcmsfiles/js/*
Allow: /hostcmsfiles/jquery/*
Allow: */upload/*.jpg
Allow: */upload/*.jpeg
Allow: */upload/*.png
Allow: */upload/*.gif

User-agent: *
Disallow: /admin
Disallow: /search
Disallow: /templates
Disallow: /showbanner
Disallow: /captcha.php
Disallow: /403
Disallow: /404
Disallow: /users/
Disallow: /*tag
Disallow: /*page
Disallow: */page/
Disallow: /catalogs/
Disallow: /compare_items/
Disallow: /*?
Disallow: /*filter=*
Disallow: /*cart/*
Disallow: /*sorting=*
Disallow: /*?comments=*
Disallow: /*?producer_id=*
Disallow: /*?on_page=*
Disallow: /*?text=*
Disallow: *.pdf
Disallow: *.xls
Disallow: *.doc
Disallow: *.ppt
Disallow: *.txt
Disallow: *utm
Disallow: *clid=
Disallow: *openstat
Disallow: *from
Allow: /hostcmsfiles/css/*
Allow: /hostcmsfiles/js/*
Allow: /hostcmsfiles/jquery/*
Allow: */upload/*.jpg
Allow: */upload/*.jpeg
Allow: */upload/*.png
Allow: */upload/*.gif

Host: https://www.site.ru
Sitemap: https://www.site.ru/sitemap.xml