diff --git a/Discovery/Web-Content/RobotsDisallowed-Top500.txt b/Discovery/Web-Content/RobotsDisallowed-Top500.txt
deleted file mode 100644
index 2ca81d91..00000000
--- a/Discovery/Web-Content/RobotsDisallowed-Top500.txt
+++ /dev/null
@@ -1,499 +0,0 @@
-
-api/
-admin/
-wp-admin/
-template/
-data/
-search/
-install/
-admin.php
-tupian
-js/
-member.php
-includes/
-uc_client/
-userapp.php?mod=app&*
-uc_server/
-tmp/
-static/
-source/
-search.php
-modules/
-misc.php
-home.php?mod=spacecp*
-forum.php?mod=redirect*
-forum.php?mod=post*
-connect.php
-config/
-api.php
-*mobile=yes*
-*?mod=misc*
-*?mod=attachment*
-themes/
-xmlrpc.php
-admin
-images/
-templates/
-plus/stow.php
-plus/search.php
-plus/rss.php
-plus/recommend.php
-plus/mytag_js.php
-plus/feedback_js.php
-plus/count.php
-cgi-bin/
-user/
-templets
-scripts/
-plus/shops_buyaction.php
-plus/posttocar.php
-plus/erraddsave.php
-plus/disdls.php
-plus/carbuyaction.php
-plus/car.php
-plus/advancedsearch.php
-plus/ad_js.php
-member/
-index.php
-include
-css/
-ajax/
-1000prosm.php
-:
-wp-includes/
-user
-search
-language/
-install.php
-cgi-bin
-cache/
-LICENSE.txt
-user/register/
-user/password/
-user/login/
-update.php
-register/
-profiles/
-plugins/
-page/
-node/add/
-misc/
-error/
-cron.php
-common/
-comment/reply/
-UPGRADE.txt
-MAINTAINERS.txt
-INSTALL.txt
-INSTALL.pgsql.txt
-INSTALL.mysql.txt
-CHANGELOG.txt
-*?*
-upload/
-tag/
-order/
-mediamainlog.php
-logs/
-lib/
-inc/
-fcmedianet.js
-commons/
-cmedianet
-cmdynet
-click.php
-category/
-__media__/js/templates.js
-?q=user/register/
-?q=user/password/
-?q=user/login/
-?q=search/
-?q=node/add/
-?q=comment/reply/
-?q=admin/
-*gclid
-*.swf
-ywidget
-wp-content/themes
-wp-content/plugins
-wp-admin
-wingBanner/
-usr-bin
-udaup.php
-txt4.php
-txt3.php
-txt2.php
-txt1.php
-tns/
-timetable
-super-product*
-static/*.html
-socs.js
-skin/
-search_ds/
-remittance/
-redirect
-recipes/
-rd2
-rd1
-primer-informers.php
-price_item.aspx
-poiskmainz2.php
-photoupload/
-payment/
-pay1000/in.php
-openapi/
-obyavi.php
-new-product*
-new-product
-new-b-product*
-new-b-product
-manage/
-login.php
-kabinet.php
-jsp/
-informer.php
-info/rules
-index
-help.php?
-help.php
-goo2.php
-goo.php
-fonts/
-firm/*/*.aspx
-estate/teaseritems
-estate/firmestate/firm
-download/
-doski/sendpismo.php
-doski/senddoski.php
-doski/
-doski.php/doski.php
-csagent/
-community/
-chatping/
-catalog/spravochnaia-aptek.aspx
-catalog/search*.aspx
-catalog/raion.aspx
-catalog/drugstore.aspx
-catalog/default.aspx
-catalog/alert.aspx
-cart
-cabinet
-blank_paket2.php
-blank_paket.php
-bitrix/
-banks/request
-b-product*
-b-product
-auth/
-auth.php
-aff-bin
-advert.php
-abuse.php
-aboutinformer2.php
-aboutinformer
-5let.php
-2012.php
-1000text.php
-1000servis.php
-1000rassil.php
-1000rassil
-1000prosm
-1000perex.php
-1000paket
-1000p.php
-*register=yes
-*print_course=Y
-*logout=yes
-*login=yes
-*forgot_password=yes
-*change_password=yes
-*bitrix_*=
-*backurl=*
-*back_url_admin=*
-*back_url=*
-*auth=yes
-*BACK_URL=*
-*BACKURL=*
-*?print=
-*&print=
-*search*
-*products
-*print
-*mark_id*
-*add*
-*/trackback
-wp-register.php
-wp-includes
-wp-content/cache
-v/
-user/logout/
-upgrade/
-tourdesc
-text
-temp/
-tag
-system/
-statistics.html
-services/
-search*
-save
-plus/
-personal/
-order-option
-opinion
-note
-message.php
-media/
-logout/
-login/
-libraries/
-languages/
-installation/
-include/
-img/
-help/
-final
-filter/tips/
-engine/go.php
-engine/download.php
-detail/
-delay
-counters
-contact/
-components/
-cert/
-catalog/
-cart/
-board/
-blocks/
-basket/
-apps/
-administrator/
-INSTALL.sqlite.txt
-?q=user/logout/
-?q=filter/tips/
-?level=12
-?
-404.html
-*x=
-*utm_source=*
-*utm_medium=*
-*utm_content=*
-*utm_campaign=*
-*utm
-*tourpoisk
-*subaction=userinfo
-*sputprtn
-*search
-*print=
-*openstat
-*level=liger*
-*img/managers/docs/
-*ggl
-*from=mail
-*do=stats
-*do=pm
-*do=lostpassword
-*do=lastcomments
-*do=feedback
-*do=addnews
-*act=viewinfo*
-*_openstat
-*WorkingVersion
-*Chld
-*?vidget
-*?mode=viewprofile
-*?from=begun
-*?fb_xd_fragment
-*?
-*/page=
-*.gif$
-espacepro*
-developpeurs/
-?&what=
-z/
-y/
-xmlrpc/
-xml/
-xls_redirect/
-wp-trackback
-wp-login.php
-wp-comments
-widget-118218/*
-webpage/
-wap/
-vodnik/tep/
-vodnik/sekret/
-vodnik/includes/
-vodnik/htdocs/
-vodnik/cgi-bin/
-vklad/id
-video/
-video-recepty
-users/
-traf/map/
-trackback
-top-sellers/
-survey.php
-subcatalog/
-store
-sms/
-signup
-signaler-erreur/*
-signaler-erreur/
-shop/
-services
-sendurl
-sendcard
-send
-search.asp
-scripts/iefix.js
-scheme
-rules.html
-route/
-respond.php
-require/
-report.html
-register/vip_pay.html
-register.php
-region.php
-referers_list.php
-redo_form/
-redirect/
-receive.php
-realty_map_data/
-quanto-vale-seu-apartamento/passo*
-pw_app.php
-pw_api.php
-pw_ajax.php
-profile.php
-profile
-product_by_id/
-product/
-pp/
-post.php
-pm
-phpmailer/
-pdd/user/stats/
-pdd/themes/
-pdd/random/
-pdd/init/
-pdd/exam/
-pdd/change/
-pda/
-parking.php4
-panier.html
-pages/displayCalculatorV2/
-owners/
-out/
-out.php
-offers/buy/
-notebooks/brand-packard%20bell/
-notebooks/brand-/
-network/
-nalog?calc
-myauto
-mst/cache/images/
-mode/
-mobile/
-members
-mail/
-loyalty/
-logout
-lang/
-kurs/undefined
-kurs/news/
-kurs/mforecast/
-kurs/RSS/
-kredit/id
-js_no_index/
-job.php
-ipdata/
-interest/
-install
-ingredients/
-infrastructure/get_geo/
-index_dev.php
-index/
-index.php/
-index$
-imprimer/*
-imprimer/
-htdocs/
-how_to_choose/95728/
-hack/
-groups
-goods_script.php
-go
-games/*/play
-game/*/play
-gallery.php
-gal/
-forumcp.php
-forum/language/
-forum/db/
-forum/cgi-bin/
-forum/cache/
-forum/admin/
-feeds/
-feed.php
-examples/my-components/
-examples/download/download_private/
-engine/ajax/
-eng
-email/
-e-store/paid/detail.php
-e-store/affiliates/
-dynamic
-dle-rules-page.html
-developpeurs/*
-deprecated_browser.html
-denunciar-anuncio/
-db/autos/*?p
-db/autos.html?r
-db/
-cycle_image.php
-customer/
-cswjjd/
-cssikexin/
-createcard
-content1
-content/links/my/
-content/board/my/
-content
-contacts/*
-complaint/
-comparemod/*_vs$
-comparemod/$
-compare/*_vs$
-communication/forum/user/
-communication/forum/search/
-communication/blog/search.php
-comments/feed/
-comments
-comment.php
-collection/
-cms/FR-FR/pneu-hiver/saison/hiver/VREDESTEIN/MICHELIN/X-ICE-XI3.html
-cms/FR-FR/pneu-hiver/saison/hiver/HANKOOK/VREDESTEIN/MICHELIN/ALPIN-5.html
-cms/FR-FR/pneu-4x4/pneu/4x4/TOYO/ATTURO/BRIDGESTONE/INSA-TURBO/BRIDGESTONE/ATTURO/AZ800.html
-cms/FR-FR/pneu-4x4/pneu/4x4/HANKOOK/HANKOOK/DYNAPRO-MT-RT03.html
-cms/FR-FR/pneu-4x4/pneu/4x4/BRIDGESTONE/BRIDGESTONE/TOYO/INSA-TURBO/HANKOOK/BF-GOODRICH/INSA-TURBO/RANGER.html
-cms/FR-FR/pneu-4x4/pneu/4x4/BF-GOODRICH/BRIDGESTONE/DUELER-A-T-694.html
-cms/FR-FR/pneu-4x4/pneu/4x4/ATTURO/BF-GOODRICH/ALL-TERRAIN-T-A-KO.html
-club/search/
-club/messages/
-club/log/
-club/group/search/
-club/gallery/tags/
-club/forum/search/
-club/$
-ck.php
-checkout
-chatgatewaycounts/
-catégorie/
-catalog/product_compare/
-cars/question/
-captcha.php
-bundles/*
-blogs
-bin.aspx?ID*