diff --git a/Discovery/Web-Content/RobotsDisallowed-Top1000.txt b/Discovery/Web-Content/RobotsDisallowed-Top1000.txt
deleted file mode 100644
index f962e954..00000000
--- a/Discovery/Web-Content/RobotsDisallowed-Top1000.txt
+++ /dev/null
@@ -1,999 +0,0 @@
-
-api/
-admin/
-wp-admin/
-template/
-data/
-search/
-install/
-admin.php
-tupian
-js/
-member.php
-includes/
-uc_client/
-userapp.php?mod=app&*
-uc_server/
-tmp/
-static/
-source/
-search.php
-modules/
-misc.php
-home.php?mod=spacecp*
-forum.php?mod=redirect*
-forum.php?mod=post*
-connect.php
-config/
-api.php
-*mobile=yes*
-*?mod=misc*
-*?mod=attachment*
-themes/
-xmlrpc.php
-admin
-images/
-templates/
-plus/stow.php
-plus/search.php
-plus/rss.php
-plus/recommend.php
-plus/mytag_js.php
-plus/feedback_js.php
-plus/count.php
-cgi-bin/
-user/
-templets
-scripts/
-plus/shops_buyaction.php
-plus/posttocar.php
-plus/erraddsave.php
-plus/disdls.php
-plus/carbuyaction.php
-plus/car.php
-plus/advancedsearch.php
-plus/ad_js.php
-member/
-index.php
-include
-css/
-ajax/
-1000prosm.php
-:
-wp-includes/
-user
-search
-language/
-install.php
-cgi-bin
-cache/
-LICENSE.txt
-user/register/
-user/password/
-user/login/
-update.php
-register/
-profiles/
-plugins/
-page/
-node/add/
-misc/
-error/
-cron.php
-common/
-comment/reply/
-UPGRADE.txt
-MAINTAINERS.txt
-INSTALL.txt
-INSTALL.pgsql.txt
-INSTALL.mysql.txt
-CHANGELOG.txt
-*?*
-upload/
-tag/
-order/
-mediamainlog.php
-logs/
-lib/
-inc/
-fcmedianet.js
-commons/
-cmedianet
-cmdynet
-click.php
-category/
-__media__/js/templates.js
-?q=user/register/
-?q=user/password/
-?q=user/login/
-?q=search/
-?q=node/add/
-?q=comment/reply/
-?q=admin/
-*gclid
-*.swf
-ywidget
-wp-content/themes
-wp-content/plugins
-wp-admin
-wingBanner/
-usr-bin
-udaup.php
-txt4.php
-txt3.php
-txt2.php
-txt1.php
-tns/
-timetable
-super-product*
-static/*.html
-socs.js
-skin/
-search_ds/
-remittance/
-redirect
-recipes/
-rd2
-rd1
-primer-informers.php
-price_item.aspx
-poiskmainz2.php
-photoupload/
-payment/
-pay1000/in.php
-openapi/
-obyavi.php
-new-product*
-new-product
-new-b-product*
-new-b-product
-manage/
-login.php
-kabinet.php
-jsp/
-informer.php
-info/rules
-index
-help.php?
-help.php
-goo2.php
-goo.php
-fonts/
-firm/*/*.aspx
-estate/teaseritems
-estate/firmestate/firm
-download/
-doski/sendpismo.php
-doski/senddoski.php
-doski/
-doski.php/doski.php
-csagent/
-community/
-chatping/
-catalog/spravochnaia-aptek.aspx
-catalog/search*.aspx
-catalog/raion.aspx
-catalog/drugstore.aspx
-catalog/default.aspx
-catalog/alert.aspx
-cart
-cabinet
-blank_paket2.php
-blank_paket.php
-bitrix/
-banks/request
-b-product*
-b-product
-auth/
-auth.php
-aff-bin
-advert.php
-abuse.php
-aboutinformer2.php
-aboutinformer
-5let.php
-2012.php
-1000text.php
-1000servis.php
-1000rassil.php
-1000rassil
-1000prosm
-1000perex.php
-1000paket
-1000p.php
-*register=yes
-*print_course=Y
-*logout=yes
-*login=yes
-*forgot_password=yes
-*change_password=yes
-*bitrix_*=
-*backurl=*
-*back_url_admin=*
-*back_url=*
-*auth=yes
-*BACK_URL=*
-*BACKURL=*
-*?print=
-*&print=
-*search*
-*products
-*print
-*mark_id*
-*add*
-*/trackback
-wp-register.php
-wp-includes
-wp-content/cache
-v/
-user/logout/
-upgrade/
-tourdesc
-text
-temp/
-tag
-system/
-statistics.html
-services/
-search*
-save
-plus/
-personal/
-order-option
-opinion
-note
-message.php
-media/
-logout/
-login/
-libraries/
-languages/
-installation/
-include/
-img/
-help/
-final
-filter/tips/
-engine/go.php
-engine/download.php
-detail/
-delay
-counters
-contact/
-components/
-cert/
-catalog/
-cart/
-board/
-blocks/
-basket/
-apps/
-administrator/
-INSTALL.sqlite.txt
-?q=user/logout/
-?q=filter/tips/
-?level=12
-?
-404.html
-*x=
-*utm_source=*
-*utm_medium=*
-*utm_content=*
-*utm_campaign=*
-*utm
-*tourpoisk
-*subaction=userinfo
-*sputprtn
-*search
-*print=
-*openstat
-*level=liger*
-*img/managers/docs/
-*ggl
-*from=mail
-*do=stats
-*do=pm
-*do=lostpassword
-*do=lastcomments
-*do=feedback
-*do=addnews
-*act=viewinfo*
-*_openstat
-*WorkingVersion
-*Chld
-*?vidget
-*?mode=viewprofile
-*?from=begun
-*?fb_xd_fragment
-*?
-*/page=
-*.gif$
-espacepro*
-developpeurs/
-?&what=
-z/
-y/
-xmlrpc/
-xml/
-xls_redirect/
-wp-trackback
-wp-login.php
-wp-comments
-widget-118218/*
-webpage/
-wap/
-vodnik/tep/
-vodnik/sekret/
-vodnik/includes/
-vodnik/htdocs/
-vodnik/cgi-bin/
-vklad/id
-video/
-video-recepty
-users/
-traf/map/
-trackback
-top-sellers/
-survey.php
-subcatalog/
-store
-sms/
-signup
-signaler-erreur/*
-signaler-erreur/
-shop/
-services
-sendurl
-sendcard
-send
-search.asp
-scripts/iefix.js
-scheme
-rules.html
-route/
-respond.php
-require/
-report.html
-register/vip_pay.html
-register.php
-region.php
-referers_list.php
-redo_form/
-redirect/
-receive.php
-realty_map_data/
-quanto-vale-seu-apartamento/passo*
-pw_app.php
-pw_api.php
-pw_ajax.php
-profile.php
-profile
-product_by_id/
-product/
-pp/
-post.php
-pm
-phpmailer/
-pdd/user/stats/
-pdd/themes/
-pdd/random/
-pdd/init/
-pdd/exam/
-pdd/change/
-pda/
-parking.php4
-panier.html
-pages/displayCalculatorV2/
-owners/
-out/
-out.php
-offers/buy/
-notebooks/brand-packard%20bell/
-notebooks/brand-/
-network/
-nalog?calc
-myauto
-mst/cache/images/
-mode/
-mobile/
-members
-mail/
-loyalty/
-logout
-lang/
-kurs/undefined
-kurs/news/
-kurs/mforecast/
-kurs/RSS/
-kredit/id
-js_no_index/
-job.php
-ipdata/
-interest/
-install
-ingredients/
-infrastructure/get_geo/
-index_dev.php
-index/
-index.php/
-index$
-imprimer/*
-imprimer/
-htdocs/
-how_to_choose/95728/
-hack/
-groups
-goods_script.php
-go
-games/*/play
-game/*/play
-gallery.php
-gal/
-forumcp.php
-forum/language/
-forum/db/
-forum/cgi-bin/
-forum/cache/
-forum/admin/
-feeds/
-feed.php
-examples/my-components/
-examples/download/download_private/
-engine/ajax/
-eng
-email/
-e-store/paid/detail.php
-e-store/affiliates/
-dynamic
-dle-rules-page.html
-developpeurs/*
-deprecated_browser.html
-denunciar-anuncio/
-db/autos/*?p
-db/autos.html?r
-db/
-cycle_image.php
-customer/
-cswjjd/
-cssikexin/
-createcard
-content1
-content/links/my/
-content/board/my/
-content
-contacts/*
-complaint/
-comparemod/*_vs$
-comparemod/$
-compare/*_vs$
-communication/forum/user/
-communication/forum/search/
-communication/blog/search.php
-comments/feed/
-comments
-comment.php
-collection/
-cms/FR-FR/pneu-hiver/saison/hiver/VREDESTEIN/MICHELIN/X-ICE-XI3.html
-cms/FR-FR/pneu-hiver/saison/hiver/HANKOOK/VREDESTEIN/MICHELIN/ALPIN-5.html
-cms/FR-FR/pneu-4x4/pneu/4x4/TOYO/ATTURO/BRIDGESTONE/INSA-TURBO/BRIDGESTONE/ATTURO/AZ800.html
-cms/FR-FR/pneu-4x4/pneu/4x4/HANKOOK/HANKOOK/DYNAPRO-MT-RT03.html
-cms/FR-FR/pneu-4x4/pneu/4x4/BRIDGESTONE/BRIDGESTONE/TOYO/INSA-TURBO/HANKOOK/BF-GOODRICH/INSA-TURBO/RANGER.html
-cms/FR-FR/pneu-4x4/pneu/4x4/BF-GOODRICH/BRIDGESTONE/DUELER-A-T-694.html
-cms/FR-FR/pneu-4x4/pneu/4x4/ATTURO/BF-GOODRICH/ALL-TERRAIN-T-A-KO.html
-club/search/
-club/messages/
-club/log/
-club/group/search/
-club/gallery/tags/
-club/forum/search/
-club/$
-ck.php
-checkout
-chatgatewaycounts/
-catégorie/
-catalog/product_compare/
-cars/question/
-captcha.php
-bundles/*
-blogs
-bin.aspx?ID*
-bbs/uc_server/
-bbs/uc_client/
-bbs/template/
-bbs/static/
-bbs/source/
-bbs/search.php
-bbs/misc.php
-bbs/member.php
-bbs/install/
-bbs/home.php?mod=spacecp*
-bbs/forum.php?mod=redirect*
-bbs/forum.php?mod=post*
-bbs/data/
-bbs/config/
-bbs/api/
-bbs/api.php
-bbs/admin.php
-basket/add/*
-bank/upload.php
-author
-attachment/
-asc/
-archive/
-appointment/
-api
-annuaire-professio/
-amy695806155/
-ajax.php
-affiche.php
-admin2.php
-admin*
-addnews.html
-active.php
-about/actions/
-ErrorLog/
-?q=logout/
-?q=contact/
-?q=*
-?msk
-?favorites
-?*
-7294873
-405.html
-13629693
-*utm_source=
-*subpart=*
-*struct_id=*
-*register
-*price_min=
-*price_max=
-*pid=
-*orderby=*
-*order=*
-*name=
-*minprice=*
-*member*
-*maxprice=*
-*level=12*
-*index.php$
-*index.php
-*image
-*from=adwords
-*filter_sended=
-*do=register
-*detail.php
-*ajax=
-*add_search=*
-*action=DELETE_FROM_COMPARE_LIST
-*action=BUY
-*action=ADD_TO_COMPARE_LIST
-*action=ADD2BASKET
-*_openstat=
-*Ya_
-*SHOWALL_1=
-*QUERY=
-*Password
-*PAGE_NAME=user_post
-*PAGE_NAME=search
-*PAGE_NAME=detail_slide_show
-*?yclid
-*?st*
-*?pm*
-*?pdg
-*?brand_
-*?adm
-*/slide_show/
-*/search/
-*/page-1/
-*/gallery/*order=*
-*/?_openstat
-*.php
-*.pdf
-*.jpg$
-*.aspx$
-*.asp*
-*&st*
-*&pm*
-*?page=1
-*?action
-*?
-*/print/
-*/from_ya
-*/feed
-*/comments
-*/brand-/*
-*/OID_*
-*/?view=*
-*/?sort=*
-*/?lang.ru
-*/?f=*
-*.swf
-*&
-you
-www2.123i.com.br
-robots.txt
-m.123i.com.br
-ex:
-arch.php
-admin.php
-笠勝電子有限公司-找工作機會-68848493.htm
-~
-zxjc/
-zulin/admin/
-zrlist.aspx
-zozm/
-zoomify/
-zhuanti/reiz/admin/
-zhongda/admin/
-zhiyucainuan/
-zhicheng/admin/
-zapomenute_prihlasovaci_udaje.html
-yzjj/
-ypxsmg/
-ypxrzg/
-ypxomg/
-ypxhmg/
-yp/my/
-yp/*.asp?*
-yp/*.asp$
-youhui_n/
-youhui/*.asp?*
-youhui/*.asp$
-ylyat/
-yjx001/
-yesterday/
-year/
-xml_feeds/
-xml
-xian/zhuanti
-xhzd/
-xd_receiver.htm
-wuaiext/
-wsauto/admin/
-write_a_review.php
-wp-trackback.php
-wp-settings.php
-wp-rss2.php
-wp-rss.php
-wp-rdf.php
-wp-pass.php
-wp-mail.php
-wp-links-opml.php
-wp-feed.php
-wp-content/plugins/
-wp-content/cache/
-wp-content/
-wp-config.php
-wp-commentsrss2.php
-wp-comments-post.php
-wp-blog-header.php
-wp-atom.php
-work1
-wishlist/
-wishlist
-wish/
-wind/
-wiki/
-widget/head/login
-why/*
-week/
-wed_n/
-wed/*.asp?*
-wed/*.asp$
-webservices/
-weborder/
-webgame/
-webchatcce
-wb/
-watch*
-wangyingran
-wagon/admin/
-w/
-vote/
-vodnik/tmp/
-vod-show-id-*-p-*.html$
-vod-show*up*.html$
-vod-show*hits*.html$
-vod-search-*
-vod-play*.html$
-vocabulary/
-visitorregister*
-visitbelgrade.rs/
-visitbelgrade.net/
-vipcard/
-vip/
-vip
-views/
-viewimg_*
-view.php
-vietnamese/--------------------------
-videos/sponsor-story/
-version-info/
-velikosrce/
-va/
-v5/weborder/
-v5/action/secweborder/
-v5/PayCenter/
-v3_specialnr/
-v3_login/qzonelogin.html
-v3_index/wrong.html
-v1_popup/*
-v1_home/*
-uuseeimg/
-users/popup/*
-users/headermenu
-users/ajax/*
-userauth/
-user/.
-user/*
-user.php
-user.asp*
-upvoted/
-uploads/
-uploadfiles/
-upload.php
-upimg/
-upfile/*
-updates/
-uncategorized/test/
-ui/
-ugc/
-ucc
-uc
-ubeogradu.rs/
-txt*
-tv_stanice_xml/
-tupush/
-tuan/*.asp?*
-tuan/*.asp$
-travel/$
-translator/
-translate/
-transfer/
-tranergy
-trackback/
-toutiaohuandeng*
-topusers/
-topusers.php
-topten
-topic/*/smart/
-top.asp
-tools07073
-tools/
-today/
-to.htm?*
-thumb.php
-thread.php
-text/*
-tentarComprar/*
-temporada/*?r&
-temporada/*?q=
-tempcache/
-tempImages/
-tel-domain-registration/
-tbxl/
-sys
-syjc/
-sydh/
-sycon/
-sycg2009/
-survey/importCarownerData.aspx
-survey/SurveyFiles
-suggestion_keywords/autocomplete_suggestion
-submit/
-submit.php
-story/sponsor-story/
-story/draft/
-story.php
-store_n/
-store/*.asp?*
-store/*.asp$
-stats/count
-stats/
-station_ok/
-static
-stars/
-ssl
-ssi/
-ssd/
-src/
-sponsors/
-spider-trap
-sou-corretor-de-imoveis/
-softeval.asp?*
-soft/previewimg.asp?softid=*
-soft/previewimg.asp
-soft/list.asp?classid=*
-soft/list.asp
-soft/download.asp?softid=*
-soft/download.asp
-so/
-skins/
-sjhd
-sitesearch.asp*
-sites/
-simple/
-signup*
-signin
-sign-out.html
-sign-in.html
-showpro.aspx
-showerr.asp
-shoppingcart.html
-shopping_cart
-shopping-item/*
-shop/shopmember/
-shop/admin/
-shitou/
-shimisi/
-shengyuan/admin/
-share/
-shangjiahudong
-sezioni/976/test-approvazione
-sezioni/949/condizioni-generali-dei-servizi-offerti
-sezioni/1423/news-estate
-sezioni/1369/palinsesto-24-30-agosto
-sezioni/1367/palinsesto-17-23-agosto
-sezioni/1366/palinsesto-10-16-agosto
-sezioni/1363/palinsesto-3-9-agosto
-sezioni/1352/html5-webradio
-sezioni/1343/mobile-layout-test
-sezioni/1320/primo-piano-app-105
-sezioni/1306/ylenia
-sezioni/1292/il-blog-di-leone-dilernia
-sezioni/1269/widget-news
-sezioni/1051/prova-diretta
-sezioni/1006/test
-servis-za-prodavce/
-services/*
-servicequery
-serverList/
-server/
-senyang/
-sendpwd.php
-seminarprojects/tag/
-sell/*?*
-selection_station_de_montage/
-securimage/
-secure/
-seasons_commu/*
-searchurl/
-searchurl.php
-search_keywords/*
-search_*
-searchProduct.ac
-search.asp?word=*
-school-lunch/*
-save_data.asp
-save-pro*
-save-biz*
-sales/
-sales
-sakura/
-s/tao/
-rss/
-rss.xml
-rss-feeds/
-rongji/admin/
-robot/
-rjs/
-reviews-Ratings
-reviews
-result/
-result.php
-resource/activity/buyingingroup/index.html
-res/
-related/*
-rel/q/
-register.html
-regAndAsk.html*
-recommended/
-recherche
-recent/
-readme.html
-read.php
-rd/
-rd.php*
-random
-qy_xx.aspx
-qy_pro.aspx
-qy_msg.aspx
-qy_about.aspx
-quisDizer/
-quero-contribuir-com-informacoes/*
-quero-comercializar-meu-apartamento/*
-qqlive.html
-qqlive.htm
-qqajax.php
-qiushengzhilu*
-qiche/*.asp?*
-qiche/*.asp$
-pwr/simplyink/pwr/6n66vqj2/rawdata/
-pwr/simplyink/pwr/6n66vqj2/debug.html
-pwr/4inkjets/pwr/n9kd9e5d/rawdata/
-pwr/4inkjets/pwr/n9kd9e5d/debug.html
-pwr/123inkjets/pwr/885jwpi5/rawdata/
-pwr/123inkjets/pwr/885jwpi5/debug.html
-pw8/
-publish/
-public_html/
-psreports/
-psp/
-psc/
-profile/
-produits/
-produits-de-bar/contact_us.php
-products/
-productreview/
-product/solutions/solutions/legal_notice/about/index.html
-product/service/service/legal_notice/about/about_us.html
-product/list.aspx
-product/legal_notice/product/supervision/company_news/index.html
-product/legal_notice/cooperation/solutions/about/supervision/product/mall.html
-product/legal_notice/cooperation/legal_notice/solutions/company_news/supervision/joinus.html
-product/legal_notice/cooperation/legal_notice/solutions/company_news/20140412v2.html
-product/legal_notice/cooperation/company_news/company_news/index.html
-product/index.aspx
-product/cooperation/company_news/service/legal_notice/index.html
-product/company_news/supervision/company_news/cooperation/policy.html
-product/company_news/service/company_news/solutions/index.html
-product/company_news/legal_notice/supervision/legal_notice/index.html
-product/company_news/legal_notice/cooperation/service/index.html
-product/company_news/cooperation/company_news/solutions/index.html
-product/company_news/about/legal_notice/supervision/index.html
-product/advertisement.html
-product.ac
-pro/*/reserva*
-pro/*/rating/*
-pro/*/ldr*
-pro/*/date*
-pro/*/contents*
-pro/*/contacts*
-pro/*/bizmedia*
-pro/*/biz*
-private
-pri_submit.asp
-post1/
-post/*?*
-post/*.asp?*
-post/*.asp$
-pomoc-a-kontakt/kontakt/
-poll/
-poisk/
-pma/
-plus/list.php?tid=2078&TotalResult=410&PageNo=
-plus/list.php?tid=2078
-plugin.php?id=*
-player/*
-pingrong/
-picture-gallery/sponsor-story/
-pic/
-pianor/
-phpsso_server
-phpcms
-phonebook/*.asp?*
-phonebook/*.asp$
-phb/jfb.htm*
-pharmacies/
-perto-de-mim/
-personal_ads/
-permalink/
-peradmin/*
-pdf/
-pdf.asp
-pay/
-parapharmacie/pharmacies/
-panier
-pages/interactives/sponsor-story/
-page_confirm.asp
-pageRight.html*
-page.php
-p.html
-organiserenquiry
-order
-optsale/last/toshiba_u500_1f4.html
-optsale/last/toshiba_u500_1dq.html
-optsale/last/toshiba_u500_18p.html
-optsale/last/toshiba_t130_16u.html
-optsale/last/toshiba_t110_12g.html
-optsale/last/toshiba_qosmio_x500_110.html